From 6a9632e2fbb7d6f8f3df0a2b3f38c44347f3b9f4 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 06:11:26 +0000 Subject: [PATCH 1/9] fix(native): recognize Hack Compose network and metadata conventions --- docs/guides/native-candidate.md | 14 +-- docs/reference/native-config-import.md | 5 +- .../src/project/compose/hack_conventions.rs | 89 ++++++++++++++ .../runtime-core/src/project/compose/mod.rs | 71 ++++++++--- .../src/project/compose/ownership_labels.rs | 7 +- .../src/project/compose/service.rs | 15 ++- .../runtime-core/src/provider/graph/config.rs | 58 ++++++++- .../runtime-core/src/provider/graph/routes.rs | 4 +- .../runtime-core/src/provider/graph/tests.rs | 5 +- src/backends/native-preflight.ts | 9 +- src/backends/native-project-routing.ts | 29 +++-- src/backends/native-project-start.ts | 12 ++ src/lib/hack-ownership-labels.ts | 8 +- src/lib/native-compose-backend-features.ts | 67 +++++------ src/lib/native-hack-compose-conventions.ts | 99 ++++++++++++++++ tests/hack-ownership-labels.test.ts | 2 +- tests/init-native-backend.test.ts | 8 +- ...ative-compose-backend-features-cli.test.ts | 82 +++++++------ tests/native-compose-backend-features.test.ts | 28 ++--- tests/native-hack-compose-conventions.test.ts | 112 ++++++++++++++++++ tests/native-project-routing.test.ts | 21 ++-- 21 files changed, 583 insertions(+), 162 deletions(-) create mode 100644 packages/runtime-core/src/project/compose/hack_conventions.rs create mode 100644 src/lib/native-hack-compose-conventions.ts create mode 100644 tests/native-hack-compose-conventions.test.ts diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index c8d2fe6fc..ee6e9b3b7 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -335,16 +335,16 @@ Compose backend. Native support for them is planned after v5. | Compose feature | Kept on the Compose backend when | | --- | --- | -| External network | A top-level network sets `external: true` or `external: {name: ...}`. A reviewed `x-hack-isolated: true` replacement is not external: it creates an owned internal bridge. A network that the selected adaptation (`HACK_NATIVE_ADAPTATION` or the committed `.hack/native/adaptation.json`; see [Native settings files](#native-settings-files)) lists in `isolatedNetworks` gets the same replacement, so it is not external either. | +| Foreign external network | Exact external names `hack-dev` and `hack-logging` map to fresh native-owned project bridges, including logical aliases; they do not attach to Docker networks or share cross-project container DNS. Native publishes routes independently of Docker’s global Caddy. Other external networks stay refused by name. A top-level network sets `external: true` or `external: {name: ...}`. A reviewed `x-hack-isolated: true` replacement is not external: it creates an owned internal bridge. A network that the selected adaptation (`HACK_NATIVE_ADAPTATION` or the committed `.hack/native/adaptation.json`; see [Native settings files](#native-settings-files)) lists in `isolatedNetworks` gets the same replacement, so it is not external either. | +| Duplicate Hack network aliases | Two logical declarations name the same `hack-dev` or `hack-logging` identity. Keep one declaration and attach its services to that declaration; native refuses instead of silently splitting connectivity. | | External volume | A top-level volume sets `external: true` or `external: {name: ...}`. The native runtime only creates volumes it owns, and there is no native replacement for an external volume. | -| Reserved ownership label | A service, network or volume label key starts with `com.docker.compose.`, `io.hack-local.`, `io.hack.native-config.`, `io.hack.native-file.` or `io.hack.storage-witness.`. Compose, the native runtime and native-config Compose use these labels to prove which resources they own. | | Build outside the basic form | A service `build` is not the basic form that [`hack config import --dry-run`](../reference/native-config-import.md#pure-basic-build-preview) accepts (a relative context and optional relative `dockerfile` and `target`), or the service also sets `image`. Build arguments, secrets, SSH, cache options, platforms and other build options are outside the basic form. With `HACK_NATIVE_BUILDS=1` (see below), only a build field other than `context`, `dockerfile`, `target` and `args`, or `image` together with `build`, counts; native build admission reviews the values of those four fields itself. | | Non-native platform | A service sets a `platform` other than `linux/arm64` or `linux/arm64/v8`, for example `linux/amd64`. The native VM runs Linux arm64. A missing or `null` `platform` is unset and passes. `linux/amd64` passes only for a service that the `HACK_NATIVE_ADAPTATION` selection opts in to [amd64 emulation](#amd64-emulation-with-rosetta-2); every other platform value stays on the Compose backend. | Native `hack up` and `hack restart` check for these features after selecting the project and before branch, environment, lifecycle, storage or runtime work. They refuse with `E_NATIVE_PROJECT_UNSUPPORTED` and one line that names each feature -found, without network or volume names, label keys, paths or platform values. +found. Foreign external network names are bounded and named; volume names, label values, paths and platform values are omitted. Nothing is started, stopped or written. Every service is checked, including services outside the selected profiles. @@ -380,8 +380,7 @@ Other labels keep their native behavior. Routing (`caddy*`) and dependency cache (`hack.dependencies.*`) labels are reviewed as before. Other ordinary labels, such as `com.example.team` or `org.opencontainers.image.*`, are plain metadata: native does not apply them to containers and reports the warning `labels_not_applied`. -Router and runtime labels outside those forms (`traefik.*`, other `caddy.*`, -`hack.*`) still refuse with `external_route_or_owner_label`. +Foreign router directives (`traefik.*` and unsupported `caddy*` keys) and unknown semantic `hack.service.*` or `hack.dependencies.*` declarations still refuse. Their bounded label keys are named; label values are omitted. Generated Hack and Compose metadata is ignored with a warning. The Compose prune label `hack.cache.disposable` grants no native cleanup authority; declare native disposable cache subpaths with `x-hack-disposable-cache` instead. A `logging:` block with the `json-file`, `local` or `none` driver is accepted and not applied (warning `logging_not_applied`): native keeps its own bounded container @@ -394,10 +393,7 @@ the label [`hack config import --dry-run`](../reference/native-config-import.md) reports as `compose_one_shot_job`, is reviewed natively as a one-shot: the review plan marks it `one_shot`, and native `up` waits for it to exit successfully instead of waiting for it to keep running. The marker adds no container label. -Only that exact key is recognized. Any other `hack.*` key, including another -`hack.service.*` key, is still refused (`external_route_or_owner_label` in the -native review), and a marker next to a reserved ownership label does not exempt -it. The native review also refuses a marker whose value is not the string +Only that exact semantic key is recognized; unknown `hack.service.*` and `hack.dependencies.*` keys stay refused. Other Hack-generated and `com.docker.compose.*` metadata is ignored with a warning, including authored ownership labels. None is copied to native resources: ownership comes only from freshly generated Engine labels. The native review also refuses a marker whose value is not the string `true` (`invalid_one_shot_label`), a one-shot with `restart: always` or `unless-stopped` (`nonterminating_one_shot`), and a one-shot that also declares routing labels (`one_shot_route_conflict`). diff --git a/docs/reference/native-config-import.md b/docs/reference/native-config-import.md index 294ba8b47..fc8d8fb1c 100644 --- a/docs/reference/native-config-import.md +++ b/docs/reference/native-config-import.md @@ -15,9 +15,8 @@ projects, read managed environment values or keys, or inspect/change engine resources. Any refusal discards the private candidate; no partial conversion is reported as successful. -An unchanged project that uses an external network (one that the -`HACK_NATIVE_ADAPTATION` selection does not list in `isolatedNetworks`), a reserved -ownership label, a build outside the basic form below, or a non-arm64 platform +An unchanged project that uses a foreign external network (other than Hack’s own +`hack-dev` and `hack-logging`, and without an `isolatedNetworks` replacement), a build outside the basic form below, or a non-arm64 platform stays on the Compose backend in v5. Native `up` refuses those features before any work, and `hack doctor` reports them; see [Compose projects that stay on the Compose backend](../guides/native-candidate.md#compose-projects-that-stay-on-the-compose-backend). diff --git a/packages/runtime-core/src/project/compose/hack_conventions.rs b/packages/runtime-core/src/project/compose/hack_conventions.rs new file mode 100644 index 000000000..952e5eed8 --- /dev/null +++ b/packages/runtime-core/src/project/compose/hack_conventions.rs @@ -0,0 +1,89 @@ +//! Pure compatibility with Hack-generated Compose metadata; never adoption authority. +use super::super::*; +use super::fields::*; +use serde_json::{Map, Value}; + +pub(super) fn network( + name: &str, + value: &Map, +) -> Result<(bool, Option), CandidateError> { + let external = match value.get("external") { + None | Some(Value::Bool(false)) => false, + Some(Value::Bool(true)) | Some(Value::Object(_)) => true, + _ => { + return Err(problem( + "invalid_network_replacement", + "External network declaration must be a boolean or named mapping.", + )); + } + }; + let legacy = value.get("external").and_then(Value::as_object); + if legacy.is_some_and(|v| v.keys().any(|key| key != "name")) { + return Err(problem( + "invalid_network_replacement", + "External network mapping accepts only its name.", + )); + } + let legacy_name = legacy.and_then(|v| v.get("name")).map(string).transpose()?; + let explicit = value.get("name").map(string).transpose()?; + if matches!((legacy_name, explicit), (Some(a), Some(b)) if a != b) { + return Err(problem( + "invalid_network_replacement", + "External network names disagree.", + )); + } + let actual = explicit.or(legacy_name).unwrap_or(name); + identifier(actual)?; + Ok(( + external, + (external && matches!(actual, "hack-dev" | "hack-logging")).then(|| actual.to_owned()), + )) +} + +pub(super) fn ignored_label(key: &str) -> bool { + key == "com.docker.compose" + || key.starts_with("com.docker.compose.") + || super::ownership_labels::is_hack_ownership_label(key) + || ((key == "hack" || key.starts_with("hack.")) + && key != "hack.dependencies" + && key != "hack.service" + && !key.starts_with("hack.dependencies.") + && !key.starts_with("hack.service.")) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + #[test] + fn only_exact_hack_network_identity_is_mapped() { + for name in ["hack-dev", "hack-logging"] { + for (logical, definition) in [ + (name, json!({"external":true})), + ("alias", json!({"external":true,"name":name})), + ("alias", json!({"external":{"name":name}})), + ] { + assert_eq!( + network(logical, definition.as_object().unwrap()).unwrap(), + (true, Some(name.to_owned())) + ); + } + assert_eq!( + network( + name, + json!({"external":true,"name":"foreign"}) + .as_object() + .unwrap() + ) + .unwrap(), + (true, None) + ); + for definition in [ + json!({"external":{"name":name,"extra":true}}), + json!({"external":{"name":name},"name":"foreign"}), + ] { + assert!(network(name, definition.as_object().unwrap()).is_err()); + } + } + } +} diff --git a/packages/runtime-core/src/project/compose/mod.rs b/packages/runtime-core/src/project/compose/mod.rs index 62f9ef684..f4584a478 100644 --- a/packages/runtime-core/src/project/compose/mod.rs +++ b/packages/runtime-core/src/project/compose/mod.rs @@ -3,6 +3,7 @@ use serde_json::{Map, Value}; mod dependency_cache; mod fields; mod graph; +mod hack_conventions; mod one_shot; mod ownership_labels; mod ports; @@ -120,11 +121,12 @@ pub(super) fn compile( }; keys( v, - &["driver", "external", "x-hack-disposable-cache"], + &["driver", "external", "x-hack-disposable-cache", "labels"], &field, &mut diagnostics, ); let disposable_caches = disposable_caches(v.get("x-hack-disposable-cache"))?; + ignored_resource_labels(v.get("labels"), &field, &mut diagnostics)?; if boolean(v.get("external"), false)? { diagnostics.push(Diagnostic::error( "external_volume", @@ -144,6 +146,7 @@ pub(super) fn compile( } } let mut networks = BTreeMap::new(); + let mut hack_networks = std::collections::BTreeSet::new(); if let Some(value) = m.get("networks") { for (name, value) in object(value)? { identifier(name)?; @@ -156,11 +159,32 @@ pub(super) fn compile( }; keys( v, - &["driver", "internal", "external", "x-hack-isolated"], + &[ + "driver", + "internal", + "external", + "x-hack-isolated", + "name", + "labels", + ], &field, &mut diagnostics, ); - let external = boolean(v.get("external"), false)?; + let (external, hack_identity) = hack_conventions::network(name, v)?; + let hack_network = hack_identity.is_some(); + if hack_identity.is_some_and(|identity| !hack_networks.insert(identity)) { + diagnostics.push(Diagnostic::error( + "duplicate_hack_network", &field, + "Several logical declarations refer to the same Hack shared network; keep one declaration rather than splitting its topology.", + )); + } + ignored_resource_labels(v.get("labels"), &field, &mut diagnostics)?; + if v.contains_key("name") && !hack_network { + diagnostics.push(Diagnostic::error( + "unsupported_network_name", &format!("{field}.name"), + "Only the exact Hack shared-network names are mapped automatically; other named network identities are not adopted.", + )); + } let isolated = boolean(v.get("x-hack-isolated"), false)?; if isolated && !external { return Err(problem( @@ -168,13 +192,19 @@ pub(super) fn compile( "x-hack-isolated requires an external network declaration to replace.", )); } - if external && !isolated { + if external && !isolated && !hack_network { diagnostics.push(Diagnostic::error( "external_network", &field, "External/global networks cannot be adopted by candidate enrollment.", )); } + if hack_network { + diagnostics.push(Diagnostic::warning( + "hack_network_mapped", &field, + "Hack's shared Compose network becomes a native-owned project-local bridge. Cross-project container DNS and Docker's global Caddy network are not shared; native publishes routes separately.", + )); + } if isolated { diagnostics.push(Diagnostic::warning( "isolated_network_replacement", @@ -256,17 +286,29 @@ const ROUTER_LABEL_WORDS: [&str; 2] = ["caddy", "traefik"]; /// Runtime and Hack namespaces: the exact key and everything under it. const RESERVED_LABEL_NAMESPACES: [&str; 2] = ["hack", "com.docker.compose"]; +fn ignored_resource_labels( + value: Option<&Value>, + field: &str, + diagnostics: &mut Vec, +) -> Result<(), CandidateError> { + if !redacted_map(value, false)?.is_empty() { + diagnostics.push(Diagnostic::warning("labels_not_applied", &format!("{field}.labels"), "Authored resource labels are metadata only and are never applied to native Engine resources.")); + } + Ok(()) +} + /// Whether a service label is plain metadata: outside every Hack ownership /// namespace, every router label family and every reserved namespace. Such a /// label is never sent to the engine (graph labels are ownership-only), so it /// cannot change a run. The TypeScript admission mirrors this rule /// (`isInertMetadataLabel` in `src/backends/native-project-routing.ts`). pub(crate) fn inert_metadata_label(key: &str) -> bool { - !ownership_labels::is_hack_ownership_label(key) - && !ROUTER_LABEL_WORDS.iter().any(|word| key.starts_with(word)) - && !RESERVED_LABEL_NAMESPACES - .iter() - .any(|prefix| key == *prefix || key.starts_with(&format!("{prefix}."))) + hack_conventions::ignored_label(key) + || (!ownership_labels::is_hack_ownership_label(key) + && !ROUTER_LABEL_WORDS.iter().any(|word| key.starts_with(word)) + && !RESERVED_LABEL_NAMESPACES + .iter() + .any(|prefix| key == *prefix || key.starts_with(&format!("{prefix}.")))) } #[cfg(test)] @@ -276,13 +318,16 @@ mod inert_label_tests { /// The same table as `tests/native-project-routing.test.ts`: the CLI /// admission and this review must agree key by key. #[test] - fn metadata_labels_are_inert_and_router_runtime_and_ownership_labels_are_not() { + fn metadata_labels_are_inert_and_semantic_router_labels_are_not() { for key in [ "other", "com.example.team", "org.opencontainers.image.source", "hackathon", "com.docker.composed", + "com.docker.compose.project", + "hack.custom", + "io.hack-local.owner", ] { assert!(inert_metadata_label(key), "{key}"); } @@ -292,12 +337,6 @@ mod inert_label_tests { "caddy_ingress_network", "caddy_0", "traefik.enable", - "hack", - "hack.custom", - "com.docker.compose", - "com.docker.compose.project", - "io.hack-local", - "io.hack-local.owner", ] { assert!(!inert_metadata_label(key), "{key}"); } diff --git a/packages/runtime-core/src/project/compose/ownership_labels.rs b/packages/runtime-core/src/project/compose/ownership_labels.rs index 198c72317..83f17383b 100644 --- a/packages/runtime-core/src/project/compose/ownership_labels.rs +++ b/packages/runtime-core/src/project/compose/ownership_labels.rs @@ -5,10 +5,9 @@ /// `src/lib/hack-ownership-labels.ts`, the source of truth; /// `tests/hack-ownership-labels.test.ts` fails when the two lists differ. /// -/// A project must not declare a label in these namespaces, because the native -/// runtime and native-config Compose trust them as ownership evidence. The -/// TypeScript classifier rejects such labels first; the candidate's own label -/// review rejects them again so a bypassed classifier cannot pass one through. +/// Authored Compose declarations in these namespaces are ignored metadata and +/// never reach Engine resources. The native runtime generates its own labels +/// and verifies those independently; an authored declaration grants nothing. pub(super) const HACK_OWNERSHIP_LABEL_NAMESPACES: [&str; 4] = [ "io.hack-local.", "io.hack.native-config.", diff --git a/packages/runtime-core/src/project/compose/service.rs b/packages/runtime-core/src/project/compose/service.rs index 73248e065..b574358e5 100644 --- a/packages/runtime-core/src/project/compose/service.rs +++ b/packages/runtime-core/src/project/compose/service.rs @@ -919,14 +919,13 @@ pub(super) fn compile( } if routing_invalid || labels.keys().any(|key| { - // Hack's ownership namespaces are never exempt: no routing, - // dependency cache or one-shot declaration may carry ownership evidence. - super::ownership_labels::is_hack_ownership_label(key) - || (!(dependency_cache.is_some() && super::dependency_cache::recognized(key)) - && !(routing.is_some() && super::routing::recognized(key)) - && !super::one_shot::recognized(key) - && !(inert_route_labels && super::routing::recognized_inert(key)) - && !super::inert_metadata_label(key)) + // Authored ownership metadata is inert; only fresh generated + // Engine labels can prove ownership. + !(dependency_cache.is_some() && super::dependency_cache::recognized(key)) + && !(routing.is_some() && super::routing::recognized(key)) + && !super::one_shot::recognized(key) + && !(inert_route_labels && super::routing::recognized_inert(key)) + && !super::inert_metadata_label(key) }) { diagnostics.push(Diagnostic::error( diff --git a/packages/runtime-core/src/provider/graph/config.rs b/packages/runtime-core/src/provider/graph/config.rs index edda8087c..189a39234 100644 --- a/packages/runtime-core/src/provider/graph/config.rs +++ b/packages/runtime-core/src/provider/graph/config.rs @@ -1123,6 +1123,63 @@ mod compose_field_tests { } } + #[test] + fn duplicate_hack_aliases_name_the_review_blocker() { + let fixture = super::super::tests::Fixture::new(); + let home = super::super::tests::Fixture::new(); + let candidate = Candidate::discover(&home.0).unwrap(); + let mut input = document(json!({"networks":["hack-dev"]})); + input["networks"] = + json!({"hack-dev":{"external":true},"edge":{"external":{"name":"hack-dev"}}}); + state::write(&fixture.0.join("compose.yaml"), &input).unwrap(); + let report = project::plan( + &candidate, + PlanOptions { + branch: None, + project: &fixture.0, + compose_file: Path::new("compose.yaml"), + profiles: &[], + }, + ) + .unwrap(); + assert!( + report + .plan + .diagnostics + .iter() + .any(|d| d.code == "duplicate_hack_network" && d.severity == "error") + ); + assert!(!report.plan.enrollment_compatible); + } + + #[test] + fn hack_compose_metadata_never_supplies_engine_ownership_or_external_networks() { + let mut input = document( + json!({"networks":["hack-dev","hack-logging"],"labels":{"com.docker.compose.project":"forged-owner","io.hack-local.owner":"forged-owner","hack.owner":"forged-owner"}}), + ); + input["networks"] = json!({"hack-dev":{"external":true},"hack-logging":{"external":true}}); + let result = prepared(&input).unwrap(); + let encoded = serde_json::to_string(&result.configs).unwrap(); + assert!(!encoded.contains("forged-owner")); + assert_ne!( + result.configs["web"]["Labels"]["io.hack-local.owner"], + "forged-owner" + ); + assert!(result.resources.contains_key("network:hack-dev")); + assert!(result.resources.contains_key("network:hack-logging")); + input["networks"]["edge"] = json!({"external":{"name":"hack-dev"}}); + assert_eq!( + prepared(&input).err().unwrap().code, + "execution_incompatible" + ); + input["networks"].as_object_mut().unwrap().remove("edge"); + input["networks"]["hack-dev"]["name"] = json!("foreign"); + assert_eq!( + prepared(&input).err().unwrap().code, + "execution_incompatible" + ); + } + #[test] fn inert_route_labels_are_admitted_without_a_route_and_partial_forms_refuse() { let inert = prepared(&document( @@ -1136,7 +1193,6 @@ mod compose_field_tests { json!({"caddy":"web.example"}), json!({"caddy.tls":"internal"}), json!({"caddy":"web.example","caddy.tls":"internal","caddy.header":"x"}), - json!({"caddy":"web.example","caddy.tls":"internal","hack.owner":"x"}), ] { let error = prepared(&document(json!({ "labels": labels }))) .err() diff --git a/packages/runtime-core/src/provider/graph/routes.rs b/packages/runtime-core/src/provider/graph/routes.rs index 31260a694..e73a651de 100644 --- a/packages/runtime-core/src/provider/graph/routes.rs +++ b/packages/runtime-core/src/provider/graph/routes.rs @@ -151,7 +151,9 @@ mod tests { literal_redacted: true, }, ); - assert!(validate_services(&owner_plan, &owner_graph, true).is_err()); + // An authored Compose owner label is ignored metadata, not route or + // Engine authority; fresh native ownership is generated separately. + assert!(validate_services(&owner_plan, &owner_graph, true).is_ok()); for hosts in [ vec![], vec!["UPPER.example".into()], diff --git a/packages/runtime-core/src/provider/graph/tests.rs b/packages/runtime-core/src/provider/graph/tests.rs index 5dbc3bb67..ffa2b413a 100644 --- a/packages/runtime-core/src/provider/graph/tests.rs +++ b/packages/runtime-core/src/provider/graph/tests.rs @@ -369,15 +369,12 @@ fn inert_logging_and_metadata_labels_are_admitted_with_review_warnings() { assert!(codes.contains(&("error".into(), "external_logging_driver".into()))); assert!(admitted.is_err()); - // Router, runtime and ownership labels are never treated as inert. + // Foreign router labels stay refused; generated metadata is inert. for key in [ "traefik.enable", "caddy.header", "caddy_ingress_network", "caddy_0", - "com.docker.compose.project", - "hack.custom", - "io.hack-local.owner", ] { let (codes, admitted) = prepare(&|web| { web["labels"] = json!({ key: "x" }); diff --git a/src/backends/native-preflight.ts b/src/backends/native-preflight.ts index 22d610937..ffba7db22 100644 --- a/src/backends/native-preflight.ts +++ b/src/backends/native-preflight.ts @@ -121,6 +121,10 @@ const USE_COMPOSE = "Run this project on the Compose backend (HACK_RUNTIME_BACKEND=compose), or remove it."; const FEATURE_FINDINGS: Readonly> = { + duplicate_hack_network: { + message: "Several logical networks refer to the same Hack shared network.", + fix: "Keep one logical declaration for each of hack-dev and hack-logging and attach its services to that declaration.", + }, external_network: { message: "The Compose file uses an external network.", fix: "Declare the network in this file, or list it under isolatedNetworks in .hack/native/adaptation.json so native replaces it with an isolated one.", @@ -179,7 +183,7 @@ const PREPARATION_FINDINGS: Readonly< unsupported_label: { field: "labels", message: - "The service has a router, runtime or Hack label native does not accept. Plain metadata labels are fine; of the caddy, traefik, hack and com.docker.compose families only the reviewed caddy routing labels, dependency cache labels and the one-shot marker are admitted.", + "The service has a foreign router label or an unsupported semantic Hack label. Generated Hack and Compose metadata is ignored; Caddy routes, dependency caches and one-shot declarations keep their own checks.", fix: "Remove those labels, or run this project on the Compose backend if a router or tool needs them.", }, }; @@ -283,6 +287,7 @@ const RUNTIME_FINDINGS: Readonly> = { fix: "Create it, or correct the path. This includes env_file targets and bind mount sources.", }, external_network: FEATURE_FINDINGS.external_network, + duplicate_hack_network: FEATURE_FINDINGS.duplicate_hack_network, external_volume: FEATURE_FINDINGS.external_volume, unsupported_volume_driver: { message: "A volume uses a driver other than local.", @@ -320,7 +325,7 @@ const RUNTIME_FINDINGS: Readonly> = { external_route_or_owner_label: { message: "The service has router or runtime labels native does not adopt, or its caddy routing labels are not in the reviewed form.", - fix: "Keep only caddy, caddy.reverse_proxy and caddy.tls for routing, and remove traefik, hack and com.docker.compose labels.", + fix: "Use caddy, caddy.reverse_proxy and caddy.tls for routing; remove unsupported router directives. Generated Hack and Compose metadata needs no adaptation.", }, external_logging_driver: { message: "The service sends logs to an external logging driver.", diff --git a/src/backends/native-project-routing.ts b/src/backends/native-project-routing.ts index b5a9649d2..f31e38022 100644 --- a/src/backends/native-project-routing.ts +++ b/src/backends/native-project-routing.ts @@ -1,9 +1,11 @@ import { HackCliError } from "../lib/cli-result.ts"; import { isRecord } from "../lib/guards.ts"; import { HACK_OWNERSHIP_LABEL_NAMESPACES } from "../lib/hack-ownership-labels.ts"; +import { isIgnoredHackComposeLabel } from "../lib/native-hack-compose-conventions.ts"; import { hasOnlyNativeCacheLabels } from "./native-project-cache.ts"; const ROUTE_LABELS = new Set(["caddy", "caddy.reverse_proxy", "caddy.tls"]); +const LABEL_KEY = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/; function refused(): Error { return new HackCliError({ code: "E_NATIVE_PROJECT_UNSUPPORTED", @@ -61,13 +63,17 @@ const RESERVED_LABEL_NAMESPACES = ["hack", "com.docker.compose"] as const; * `packages/runtime-core/src/project/compose/mod.rs`. */ export function isInertMetadataLabel(key: string): boolean { - return !( - HACK_OWNERSHIP_LABEL_NAMESPACES.some( - (namespace) => key.startsWith(namespace) || key === namespace.slice(0, -1) - ) || - ROUTER_LABEL_WORDS.some((word) => key.startsWith(word)) || - RESERVED_LABEL_NAMESPACES.some( - (prefix) => key === prefix || key.startsWith(`${prefix}.`) + return ( + isIgnoredHackComposeLabel(key) || + !( + HACK_OWNERSHIP_LABEL_NAMESPACES.some( + (namespace) => + key.startsWith(namespace) || key === namespace.slice(0, -1) + ) || + ROUTER_LABEL_WORDS.some((word) => key.startsWith(word)) || + RESERVED_LABEL_NAMESPACES.some( + (prefix) => key === prefix || key.startsWith(`${prefix}.`) + ) ) ); } @@ -85,6 +91,15 @@ export function hasOnlyNativeSupportedLabels(labels: unknown): boolean { ); } +/** Only bounded label keys are named in refusals; label values are never printed. */ +export function unsupportedNativeLabelKeys(labels: unknown): readonly string[] { + return (labelEntries(labels) ?? []) + .filter(([key, value]) => !hasOnlyNativeSupportedLabels({ [key]: value })) + .map(([key]) => (LABEL_KEY.test(key) ? key : "[invalid-label]")) + .slice(0, 32) + .sort(); +} + /** Log drivers that keep logs in the engine; native reviews them and applies none. */ const LOCAL_LOGGING_DRIVERS = new Set(["json-file", "local", "none"]); diff --git a/src/backends/native-project-start.ts b/src/backends/native-project-start.ts index e176ac535..fb8f6998d 100644 --- a/src/backends/native-project-start.ts +++ b/src/backends/native-project-start.ts @@ -135,6 +135,7 @@ import { hasOnlyNativeSupportedLogging, nativeBridgeCapacity, reviewedNativeRoutes, + unsupportedNativeLabelKeys, } from "./native-project-routing.ts"; import { beginNativeProjectStartupObservation, @@ -649,6 +650,17 @@ export function prepareNativeProjectServices( } const result: Record> = {}; for (const [name, value] of Object.entries(compose.services)) { + if (isRecord(value)) { + const unsupported = unsupportedNativeLabelKeys(value.labels).filter( + (key) => key.startsWith("caddy") || key.startsWith("traefik") + ); + if (unsupported.length) { + throw new HackCliError({ + code: "E_NATIVE_PROJECT_UNSUPPORTED", + message: `Native service labels are unsupported: ${unsupported.join(", ")}. Remove the unsupported router directive; run hack doctor --native-preflight for every blocker. Label values were omitted; nothing was started.`, + }); + } + } if ( !isRecord(value) || nativeServicePreparationProblems({ diff --git a/src/lib/hack-ownership-labels.ts b/src/lib/hack-ownership-labels.ts index 2bfd5e94a..c7cac0560 100644 --- a/src/lib/hack-ownership-labels.ts +++ b/src/lib/hack-ownership-labels.ts @@ -17,10 +17,10 @@ * - `io.hack.storage-witness.`: native-config storage witness carriers * (`owner`, `generation`, `helper`, `carrier`). * - * A project must not declare a label in these namespaces, because their - * consumers trust them as ownership evidence. The TypeScript Compose - * classifier rejects them, and runtime-core's own label review rejects them - * again from a Rust mirror of this list + * Engine labels in these namespaces are ownership evidence. Authored Compose + * declarations are ignored metadata and never copied to Engine resources; + * native generates and verifies its own labels independently. Runtime-core + * mirrors this list * (`packages/runtime-core/src/project/compose/ownership_labels.rs`) that the * same test keeps identical. Add a namespace here, and to the mirror, before * writing a label in it. diff --git a/src/lib/native-compose-backend-features.ts b/src/lib/native-compose-backend-features.ts index c69b69917..1ca0b04ac 100644 --- a/src/lib/native-compose-backend-features.ts +++ b/src/lib/native-compose-backend-features.ts @@ -10,6 +10,11 @@ import { HackCliError } from "./cli-result.ts"; import { isRecord } from "./guards.ts"; import { HACK_OWNERSHIP_LABEL_NAMESPACES } from "./hack-ownership-labels.ts"; import { mapLegacyComposeBuild } from "./native-config-import-build.ts"; +import { + duplicateHackComposeNetworks, + foreignComposeNetworkNames, + isHackExternalNetwork, +} from "./native-hack-compose-conventions.ts"; /** * Compose features that keep an unchanged Compose project on the Compose @@ -18,8 +23,8 @@ import { mapLegacyComposeBuild } from "./native-config-import-build.ts"; * before any runtime request, storage preparation, lifecycle hook or stop. * * The set is closed and ordered; it is part of the refusal and the doctor - * report contract. Messages and reports name only these features, never a - * network, volume, label, path, platform or other authored value. + * report contract. Refusals name bounded foreign network identifiers. Label values, host paths + * and environment values are never included. * * Only `.hack/docker-compose.yml` itself is classified, after the explicit * `HACK_NATIVE_ADAPTATION` selection. Compose override files, `extends`, @@ -30,8 +35,9 @@ import { mapLegacyComposeBuild } from "./native-config-import-build.ts"; */ export const COMPOSE_BACKEND_FEATURES = [ "external_network", + "duplicate_hack_network", "external_volume", - "reserved_label", + "reserved_label", // Older reports only; authored metadata no longer emits this feature. "unsupported_build", "foreign_platform", ] as const; @@ -40,8 +46,8 @@ export const COMPOSE_BACKEND_FEATURES = [ export type ComposeBackendFeature = (typeof COMPOSE_BACKEND_FEATURES)[number]; /** - * Label namespaces that prove resource ownership. A project must not declare - * them, because their consumers trust them as ownership evidence: + * Engine label namespaces that prove resource ownership. Authored Compose + * declarations are ignored metadata and never copied to native Engine resources: * * - `com.docker.compose.*`: Docker Compose's own project, service and one-off * labels, which the Compose backend reads to find a project's containers. @@ -51,7 +57,7 @@ export type ComposeBackendFeature = (typeof COMPOSE_BACKEND_FEATURES)[number]; * Other labels, including Hack's routing (`caddy*`), dependency cache * (`hack.dependencies.*`) and one-shot (`hack.service.one-shot`) markers, are not * reserved: native `up` reviews them itself. The native review accepts only the - * exact one-shot key, as a completion role, and still refuses other `hack.*` keys. + * exact one-shot key as a completion role; unsupported semantic service/cache keys still refuse. */ export const RESERVED_OWNERSHIP_LABEL_PREFIXES = [ "com.docker.compose.", @@ -76,6 +82,7 @@ const COMPOSE_LIMIT = 256 * 1024; const FEATURE_PHRASES: Readonly> = { external_network: "an external Compose network", + duplicate_hack_network: "multiple logical aliases of one Hack shared network", external_volume: "an external Compose volume", reserved_label: `a label in a reserved ownership namespace (${reservedNamespaceList()})`, unsupported_build: @@ -102,24 +109,6 @@ export type ComposeBackendFeatureOptions = { readonly buildsEnabled?: boolean; }; -function labelKeys(labels: unknown): readonly string[] { - if (isRecord(labels)) { - return Object.keys(labels); - } - if (Array.isArray(labels)) { - return labels - .filter((label): label is string => typeof label === "string") - .map((label) => label.split("=", 1)[0] ?? ""); - } - return []; -} - -function hasReservedLabel(labels: unknown): boolean { - return labelKeys(labels).some((key) => - RESERVED_OWNERSHIP_LABEL_PREFIXES.some((prefix) => key.startsWith(prefix)) - ); -} - /** * Compose treats `external: true` and the legacy `external: {name: ...}` form * as a network the project does not own. The runtime's reviewed @@ -249,6 +238,9 @@ export function findComposeBackendFeatures( return []; } const found = new Set(); + if (duplicateHackComposeNetworks(compose).length) { + found.add("duplicate_hack_network"); + } const services = topLevelEntries(compose, "services").filter(isRecord); const isolated = opts?.isolatedNetworks ?? new Set(); const networks = isRecord(compose.networks) @@ -256,7 +248,9 @@ export function findComposeBackendFeatures( : []; if ( networks.some( - ([name, network]) => !isolated.has(name) && isExternalNetwork(network) + ([name, network]) => + !(isolated.has(name) || isHackExternalNetwork(name, network)) && + isExternalNetwork(network) ) ) { found.add("external_network"); @@ -264,16 +258,6 @@ export function findComposeBackendFeatures( if (topLevelEntries(compose, "volumes").some(isExternalVolume)) { found.add("external_volume"); } - const resources = [ - ...topLevelEntries(compose, "networks"), - ...topLevelEntries(compose, "volumes"), - ].filter(isRecord); - if ( - services.some((service) => hasReservedLabel(service.labels)) || - resources.some((resource) => hasReservedLabel(resource.labels)) - ) { - found.add("reserved_label"); - } const buildsEnabled = opts?.buildsEnabled === true; if (services.some((service) => isUnsupportedBuild(service, buildsEnabled))) { found.add("unsupported_build"); @@ -622,12 +606,16 @@ export function composeBackendRefusal( opts: { readonly features: readonly ComposeBackendFeature[]; readonly restart: boolean; + readonly foreignNetworks?: readonly string[]; } & ComposeBackendFeatureOptions ): HackCliError { const command = opts.restart ? "hack restart" : "hack up"; + const names = opts.foreignNetworks?.length + ? ` Foreign external networks: ${opts.foreignNetworks.join(", ")}.` + : ""; return new HackCliError({ code: "E_NATIVE_PROJECT_UNSUPPORTED", - message: `Native \`${command}\` does not support ${describeComposeBackendFeatures(opts.features, opts)}; this project runs on the Compose backend. ${COMPOSE_BACKEND_SELECTION} and run the command again. Nothing was started or stopped.`, + message: `Native \`${command}\` does not support ${describeComposeBackendFeatures(opts.features, opts)}; this project runs on the Compose backend. ${COMPOSE_BACKEND_SELECTION} and run the command again. Nothing was started or stopped.${names}`, detail: { backend: "compose", features: [...opts.features] }, }); } @@ -653,6 +641,11 @@ export async function refuseNativeComposeBackendFeatures( buildsEnabled: opts.buildsEnabled, }); if (features.length > 0) { - throw composeBackendRefusal({ ...opts, features }); + const foreignNetworks = features.includes("external_network") + ? foreignComposeNetworkNames( + await readNativeComposeDocument(opts.composeFile) + ) + : []; + throw composeBackendRefusal({ ...opts, features, foreignNetworks }); } } diff --git a/src/lib/native-hack-compose-conventions.ts b/src/lib/native-hack-compose-conventions.ts new file mode 100644 index 000000000..b5233dc68 --- /dev/null +++ b/src/lib/native-hack-compose-conventions.ts @@ -0,0 +1,99 @@ +import { isRecord } from "./guards.ts"; +import { HACK_OWNERSHIP_LABEL_NAMESPACES } from "./hack-ownership-labels.ts"; + +const NETWORK_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/; + +/** Logical Hack entry points become new native-owned bridges, never Docker attachments. */ +export function isHackExternalNetwork(name: string, value: unknown): boolean { + if (!isRecord(value)) { + return false; + } + const external = value.external; + if (!(external === true || isRecord(external))) { + return false; + } + if ( + isRecord(external) && + Object.keys(external).some((key) => key !== "name") + ) { + return false; + } + const legacy = isRecord(external) ? external.name : undefined; + if ( + legacy !== undefined && + value.name !== undefined && + legacy !== value.name + ) { + return false; + } + const actual = value.name ?? legacy ?? name; + return actual === "hack-dev" || actual === "hack-logging"; +} + +/** Authored metadata never supplies Engine ownership. Semantic Hack declarations remain reviewed. */ +export function isIgnoredHackComposeLabel(key: string): boolean { + return ( + key.startsWith("com.docker.compose.") || + key === "com.docker.compose" || + HACK_OWNERSHIP_LABEL_NAMESPACES.some( + (prefix) => key.startsWith(prefix) || key === prefix.slice(0, -1) + ) || + ((key === "hack" || key.startsWith("hack.")) && + key !== "hack.dependencies" && + key !== "hack.service" && + !key.startsWith("hack.dependencies.") && + !key.startsWith("hack.service.")) + ); +} + +/** Bounded declared foreign names for an actionable refusal; no label values or host paths. */ +export function foreignComposeNetworkNames(compose: unknown): string[] { + if (!(isRecord(compose) && isRecord(compose.networks))) { + return []; + } + return Object.entries(compose.networks) + .flatMap(([name, network]) => { + if ( + !isRecord(network) || + isHackExternalNetwork(name, network) || + network["x-hack-isolated"] === true + ) { + return []; + } + const legacy = isRecord(network.external) ? network.external : undefined; + if (!(network.external === true || legacy)) { + return []; + } + const actual = network.name ?? legacy?.name ?? name; + return typeof actual === "string" && NETWORK_NAME.test(actual) + ? [actual] + : []; + }) + .slice(0, 32) + .sort(); +} + +/** Refuse an ambiguous alias topology rather than silently splitting one Compose network. */ +export function duplicateHackComposeNetworks(compose: unknown): string[] { + if (!(isRecord(compose) && isRecord(compose.networks))) { + return []; + } + const seen = new Set(); + const duplicates = new Set(); + for (const [name, definition] of Object.entries(compose.networks)) { + if (!(isRecord(definition) && isHackExternalNetwork(name, definition))) { + continue; + } + const legacy = isRecord(definition.external) + ? definition.external.name + : undefined; + const identity = definition.name ?? legacy ?? name; + if (typeof identity === "string") { + if (seen.has(identity)) { + duplicates.add(identity); + } + seen.add(identity); + } + } + return [...duplicates].sort(); +} diff --git a/tests/hack-ownership-labels.test.ts b/tests/hack-ownership-labels.test.ts index 47901d64c..ac5202668 100644 --- a/tests/hack-ownership-labels.test.ts +++ b/tests/hack-ownership-labels.test.ts @@ -85,7 +85,7 @@ async function rustMirrorNamespaces(): Promise { return literals; } -test("runtime-core's label review rejects exactly the declared ownership namespaces", async () => { +test("runtime-core recognizes exactly the declared Engine ownership namespaces", async () => { expect(await rustMirrorNamespaces()).toEqual([ ...HACK_OWNERSHIP_LABEL_NAMESPACES, ]); diff --git a/tests/init-native-backend.test.ts b/tests/init-native-backend.test.ts index b6d79e52a..c735adaa0 100644 --- a/tests/init-native-backend.test.ts +++ b/tests/init-native-backend.test.ts @@ -118,9 +118,7 @@ test("the Compose scaffold is unchanged when the backend is compose or omitted", expect(parseCompose(omitted).networks["hack-dev"]).toEqual({ external: true, }); - expect(findComposeBackendFeatures(parseCompose(omitted))).toEqual([ - "external_network", - ]); + expect(findComposeBackendFeatures(parseCompose(omitted))).toEqual([]); }); test("the native scaffold isolates the ingress network and probes each route at its upstream port", () => { @@ -282,9 +280,7 @@ test("hack init --auto on the Compose backend keeps the Compose scaffold and nex const yaml = await Bun.file(join(repo, ".hack", "docker-compose.yml")).text(); expect(yaml).not.toContain("x-hack"); - expect(findComposeBackendFeatures(parseCompose(yaml))).toEqual([ - "external_network", - ]); + expect(findComposeBackendFeatures(parseCompose(yaml))).toEqual([]); }, 30_000); test("hack init --auto refuses an existing .hack/ with a coded line, no stack and non-interactive next steps (HACK-2069)", async () => { diff --git a/tests/native-compose-backend-features-cli.test.ts b/tests/native-compose-backend-features-cli.test.ts index 39c179dd0..c83ee430c 100644 --- a/tests/native-compose-backend-features-cli.test.ts +++ b/tests/native-compose-backend-features-cli.test.ts @@ -14,6 +14,7 @@ import { join, relative, resolve } from "node:path"; import { nativeBuildsDisabledRefusal } from "../src/backends/native-project-builds.ts"; import { NATIVE_CONFIGURATION_REFUSAL } from "../src/backends/native-project-start.ts"; import { PROJECT_ENV_KEY_FILENAME } from "../src/constants.ts"; +import { foreignComposeNetworkNames } from "../src/lib/native-hack-compose-conventions.ts"; import { setProjectEnvValue } from "../src/lib/project-env-config.ts"; import { privateCliEnv } from "./helpers/private-cli-env.ts"; @@ -41,8 +42,6 @@ const ADMITTED_START_REQUESTS = [ "project plan", "runtime up", ]; -const RESERVED_LABEL = - "a label in a reserved ownership namespace (com.docker.compose.*, io.hack-local.*, io.hack.native-config.*, io.hack.native-file.* or io.hack.storage-witness.*)"; const SELECTION = "this project runs on the Compose backend. Set HACK_RUNTIME_BACKEND=compose (hack-next and hack-v5 accept it too) and run the command again. Nothing was started or stopped."; @@ -199,13 +198,17 @@ async function expectComposeBackendRefusal(opts: { readonly secrets?: readonly string[]; }): Promise { const f = await fixture(opts.compose); + const names = foreignComposeNetworkNames(opts.compose); + const networkDetail = names.length + ? ` Foreign external networks: ${names.join(", ")}.` + : ""; // Building the child env creates the private HOME and Docker client config. f.env({}); const before = await tree(f.root); const result = await cli(f, opts.args ?? ["up"], opts.env); expectOneClosedLine( result, - `E_NATIVE_PROJECT_UNSUPPORTED: Native \`${opts.command ?? "hack up"}\` does not support ${opts.phrase}; ${SELECTION}` + `E_NATIVE_PROJECT_UNSUPPORTED: Native \`${opts.command ?? "hack up"}\` does not support ${opts.phrase}; ${SELECTION}${networkDetail}` ); for (const secret of opts.secrets ?? []) { expect(result.output).not.toContain(secret); @@ -223,45 +226,45 @@ test( networks: { "shared-dev-net": { external: true } }, }, phrase: "an external Compose network", - secrets: ["shared-dev-net"], }); }, SPAWN_TIMEOUT_MS ); test( - "a reserved ownership label refuses native up before any request or write", + "generated ownership labels pass admission without exposing or borrowing their values", async () => { - await expectComposeBackendRefusal({ - compose: { + for (const key of [ + "io.hack-local.owner", + "io.hack.storage-witness.owner", + "com.docker.compose.project", + ]) { + const f = await fixture({ services: { - web: { - image: IMAGE, - labels: { "io.hack-local.owner": "borrowed-owner-token" }, - }, + web: { image: IMAGE, labels: { [key]: "borrowed-owner-token" } }, }, - }, - phrase: RESERVED_LABEL, - secrets: ["borrowed-owner-token", "io.hack-local.owner"], - }); + }); + const result = await cli(f, ["up"]); + expect(result.output).not.toContain("borrowed-owner-token"); + expect(result.output).not.toContain("E_NATIVE_PROJECT_UNSUPPORTED"); + expect(await f.calls()).toEqual(ADMITTED_START_REQUESTS); + } }, SPAWN_TIMEOUT_MS ); test( - "a storage witness ownership label refuses native up before any request or write", + "two aliases of one Hack network refuse before a request or write", async () => { await expectComposeBackendRefusal({ compose: { - services: { - web: { - image: IMAGE, - labels: { "io.hack.storage-witness.owner": "borrowed-witness" }, - }, + services: { web: { image: IMAGE, networks: ["hack-dev"] } }, + networks: { + "hack-dev": { external: true }, + edge: { external: { name: "hack-dev" } }, }, }, - phrase: RESERVED_LABEL, - secrets: ["borrowed-witness", "io.hack.storage-witness.owner"], + phrase: "multiple logical aliases of one Hack shared network", }); }, SPAWN_TIMEOUT_MS @@ -330,7 +333,8 @@ test( volumes: { cache: { external: true } }, }, env: {}, - phrase: `an external Compose network, an external Compose volume, ${RESERVED_LABEL}, a Compose build outside the basic context, dockerfile and target form and a service platform other than linux/arm64`, + phrase: + "an external Compose network, an external Compose volume, a Compose build outside the basic context, dockerfile and target form and a service platform other than linux/arm64", }); }, SPAWN_TIMEOUT_MS @@ -365,6 +369,7 @@ test( }, }, { image: IMAGE, labels: ["com.example.team=web"] }, + { image: IMAGE, labels: { "hack.custom": "x" } }, { image: IMAGE, logging: { driver: "json-file" } }, { image: IMAGE, logging: { options: { "max-size": "10m" } } }, { image: IMAGE, logging: { driver: "none" } }, @@ -388,10 +393,6 @@ test( { image: IMAGE, labels: { "traefik.enable": "true" } }, NATIVE_CONFIGURATION_REFUSAL, ], - [ - { image: IMAGE, labels: { "hack.custom": "x" } }, - NATIVE_CONFIGURATION_REFUSAL, - ], [ { image: IMAGE, labels: { "caddy.handle": "x" } }, NATIVE_CONFIGURATION_REFUSAL, @@ -406,7 +407,14 @@ test( ] as const) { const f = await fixture({ services: { web: service } }); const result = await cli(f, ["up"]); - expectOneClosedLine(result, `E_NATIVE_PROJECT_UNSUPPORTED: ${refusal}`); + const label = + typeof service === "object" && "labels" in service + ? Object.keys(service.labels)[0] + : undefined; + const expected = label + ? `Native service labels are unsupported: ${label}. Remove the unsupported router directive; run hack doctor --native-preflight for every blocker. Label values were omitted; nothing was started.` + : refusal; + expectOneClosedLine(result, `E_NATIVE_PROJECT_UNSUPPORTED: ${expected}`); expect(result.output).not.toContain("HACK_RUNTIME_BACKEND=compose"); // The check passed: native up went on to its own source preflight. expect(await f.calls()).toEqual(["runtime check-project-share"]); @@ -517,10 +525,10 @@ test( SPAWN_TIMEOUT_MS ); -/** The documented `isolatedNetworks` adaptation for an external `hack-dev`. */ +/** The documented `isolatedNetworks` adaptation for an external `foreign-dev`. */ const HACK_DEV_COMPOSE = { - services: { web: { image: IMAGE, networks: ["hack-dev"] } }, - networks: { "hack-dev": { external: true } }, + services: { web: { image: IMAGE, networks: ["foreign-dev"] } }, + networks: { "foreign-dev": { external: true } }, }; async function adaptation(f: Fixture, selection: unknown): Promise { @@ -543,7 +551,7 @@ test( const f = await fixture(HACK_DEV_COMPOSE); const path = await adaptation(f, { version: 1, - isolatedNetworks: ["hack-dev"], + isolatedNetworks: ["foreign-dev"], }); const result = await cli(f, ["up"], { HACK_NATIVE_SHARED_SOURCE: "1", @@ -566,7 +574,7 @@ test( const f = await fixture(HACK_DEV_COMPOSE); const path = await adaptation(f, { version: 1, - isolatedNetworks: ["hack-dev"], + isolatedNetworks: ["foreign-dev"], }); const result = await cli(f, ["restart"], { HACK_NATIVE_SHARED_SOURCE: "1", @@ -598,7 +606,7 @@ test( }); expectOneClosedLine( result, - `E_NATIVE_PROJECT_UNSUPPORTED: Native \`hack up\` does not support an external Compose network; ${SELECTION}` + `E_NATIVE_PROJECT_UNSUPPORTED: Native \`hack up\` does not support an external Compose network; ${SELECTION} Foreign external networks: foreign-dev.` ); expect(await f.calls()).toEqual([]); expect(await tree(f.root)).toEqual(before); @@ -613,7 +621,7 @@ test( f.env({}); const path = await adaptation(f, { version: 2, - isolatedNetworks: ["hack-dev"], + isolatedNetworks: ["foreign-dev"], }); const before = await tree(f.root); const result = await cli(f, ["up"], { @@ -636,7 +644,7 @@ test( const f = await fixture(HACK_DEV_COMPOSE); const path = await adaptation(f, { version: 1, - isolatedNetworks: ["hack-dev"], + isolatedNetworks: ["foreign-dev"], }); const result = await cli(f, ["doctor", "--json"], { HACK_NATIVE_ADAPTATION: path, diff --git a/tests/native-compose-backend-features.test.ts b/tests/native-compose-backend-features.test.ts index 34fd35907..f3206e42b 100644 --- a/tests/native-compose-backend-features.test.ts +++ b/tests/native-compose-backend-features.test.ts @@ -95,7 +95,7 @@ test("the reserved set is Compose's namespace plus every Hack ownership namespac ]); }); -test("each reserved ownership namespace is blocking in map and list labels", () => { +test("authored ownership metadata is ignored in map and list labels", () => { for (const key of [ "com.docker.compose.project", "io.hack-local.owner", @@ -108,18 +108,18 @@ test("each reserved ownership namespace is blocking in map and list labels", () findComposeBackendFeatures( services({ image: IMAGE, labels: { [key]: "x" } }) ) - ).toEqual(["reserved_label"]); + ).toEqual([]); expect( findComposeBackendFeatures( services({ image: IMAGE, labels: [`${key}=x`] }) ) - ).toEqual(["reserved_label"]); + ).toEqual([]); expect( findComposeBackendFeatures({ ...services({ image: IMAGE }), volumes: { data: { labels: { [key]: "x" } } }, }) - ).toEqual(["reserved_label"]); + ).toEqual([]); } }); @@ -265,7 +265,6 @@ test("features come back once each in closed order across services", () => { ).toEqual([ "external_network", "external_volume", - "reserved_label", "unsupported_build", "foreign_platform", ]); @@ -295,7 +294,8 @@ test("the refusal is one coded line with closed detail and no values", async () "Native `hack up` does not support an external Compose network and a service platform other than linux/arm64; this project runs on the Compose backend. Set HACK_RUNTIME_BACKEND=compose" ); expect(error.message).not.toContain("\n"); - for (const value of ["private-net", "secret-tag", "linux/amd64", root]) { + expect(error.message).toContain("Foreign external networks: private-net."); + for (const value of ["secret-tag", "linux/amd64", root]) { expect(error.message).not.toContain(value); } expect(error.detail).toEqual({ @@ -485,7 +485,7 @@ test("the Compose file is classified after a valid HACK_NATIVE_ADAPTATION select const composeFile = join(root, "docker-compose.yml"); await writeFile( composeFile, - "services:\n web:\n image: app\n networks: [hack-dev]\nnetworks:\n hack-dev:\n external: true\n" + "services:\n web:\n image: app\n networks: [foreign-dev]\nnetworks:\n foreign-dev:\n external: true\n" ); const selection = async (name: string, value: unknown) => { const path = join(root, name); @@ -494,7 +494,7 @@ test("the Compose file is classified after a valid HACK_NATIVE_ADAPTATION select }; const isolating = await selection("isolating.json", { version: 1, - isolatedNetworks: ["hack-dev"], + isolatedNetworks: ["foreign-dev"], }); expect( await readComposeBackendFeatures(composeFile, { adaptationFile: isolating }) @@ -520,15 +520,15 @@ test("an invalid adaptation refuses as native start does, and is read only for a const root = await mkdtemp(join(tmpdir(), "compose-backend-")); roots.push(root); const external = join(root, "external.yml"); - await writeFile(external, "networks:\n hack-dev:\n external: true\n"); + await writeFile(external, "networks:\n foreign-dev:\n external: true\n"); const owned = join(root, "owned.yml"); await writeFile(owned, "services:\n web:\n image: app\n"); const invalid: unknown[] = [ - { version: 2, isolatedNetworks: ["hack-dev"] }, - { version: 1, isolatedNetworks: "hack-dev" }, - { version: 1, isolatedNetworks: ["hack-dev", "hack-dev"] }, + { version: 2, isolatedNetworks: ["foreign-dev"] }, + { version: 1, isolatedNetworks: "foreign-dev" }, + { version: 1, isolatedNetworks: ["foreign-dev", "foreign-dev"] }, { version: 1, isolatedNetworks: [7] }, - ["hack-dev"], + ["foreign-dev"], ]; for (const [index, value] of invalid.entries()) { const path = join(root, `invalid-${index}.json`); @@ -547,7 +547,7 @@ test("an invalid adaptation refuses as native start does, and is read only for a adaptNativeProject({ input: adaptationInput({ services: { web: { image: IMAGE } }, - networks: { "hack-dev": { external: true } }, + networks: { "foreign-dev": { external: true } }, }), selection: value, }) diff --git a/tests/native-hack-compose-conventions.test.ts b/tests/native-hack-compose-conventions.test.ts new file mode 100644 index 000000000..45fbafa6c --- /dev/null +++ b/tests/native-hack-compose-conventions.test.ts @@ -0,0 +1,112 @@ +import { expect, test } from "bun:test"; +import { renderCompose } from "../src/init/compose.ts"; +import { findComposeBackendFeatures } from "../src/lib/native-compose-backend-features.ts"; +import { + duplicateHackComposeNetworks, + isHackExternalNetwork, + isIgnoredHackComposeLabel, +} from "../src/lib/native-hack-compose-conventions.ts"; + +for (const name of ["hack-dev", "hack-logging"]) { + test(`maps exact ${name} identity rather than trusting a logical alias`, () => { + expect(isHackExternalNetwork(name, { external: true })).toBe(true); + expect(isHackExternalNetwork("edge", { external: true, name })).toBe(true); + expect(isHackExternalNetwork("edge", { external: { name } })).toBe(true); + expect( + isHackExternalNetwork(name, { external: true, name: "foreign" }) + ).toBe(false); + expect(isHackExternalNetwork(name, { external: { name: "foreign" } })).toBe( + false + ); + expect( + isHackExternalNetwork(name, { external: { name, other: "ignored" } }) + ).toBe(false); + expect( + isHackExternalNetwork(name, { external: { name }, name: "foreign" }) + ).toBe(false); + }); +} +test("only metadata is ignored; semantic Hack labels and routers stay reviewed", () => { + for (const key of [ + "com.docker.compose.project", + "io.hack-local.owner", + "io.hack.native-config.owner", + "hack.project", + "hack.custom", + ]) { + expect(isIgnoredHackComposeLabel(key)).toBe(true); + } + for (const key of [ + "hack.dependencies.lockfiles", + "hack.dependencies.unknown", + "hack.service.one-shot", + "hack.service.one-shot.extra", + "caddy", + "caddy.reverse_proxy", + "traefik.enable", + ]) { + expect(isIgnoredHackComposeLabel(key)).toBe(false); + } +}); +test("the real init Compose renderer needs no network or metadata adaptation", () => { + const document = Bun.YAML.parse( + renderCompose({ + name: "native-conventions", + services: [ + { + name: "web", + role: "http", + image: "example/web:1", + workingDir: "/app", + command: "serve", + env: new Map(), + labels: new Map([ + ["caddy", "web.example.hack"], + ["caddy.reverse_proxy", "{{upstreams 3000}}"], + ["caddy.tls", "internal"], + ]), + networks: ["hack-dev", "default"], + httpPort: 3000, + }, + ], + }) + ); + expect(findComposeBackendFeatures(document)).toEqual([]); + expect( + findComposeBackendFeatures({ + services: { + web: { + image: "example/web:1", + labels: { + "com.docker.compose.project": "old-project", + "io.hack-local.owner": "forged", + }, + }, + }, + networks: { + "hack-dev": { external: true }, + "hack-logging": { external: true }, + }, + }) + ).toEqual([]); + expect( + findComposeBackendFeatures({ + networks: { "hack-dev": { external: true, name: "company-private" } }, + }) + ).toEqual(["external_network"]); +}); + +test("two aliases of one Hack network refuse instead of splitting connectivity", () => { + for (const name of ["hack-dev", "hack-logging"]) { + const document = { + networks: { + [name]: { external: true }, + alias: { external: { name } }, + }, + }; + expect(duplicateHackComposeNetworks(document)).toEqual([name]); + expect(findComposeBackendFeatures(document)).toEqual([ + "duplicate_hack_network", + ]); + } +}); diff --git a/tests/native-project-routing.test.ts b/tests/native-project-routing.test.ts index b1ec3f18e..6e26452e9 100644 --- a/tests/native-project-routing.test.ts +++ b/tests/native-project-routing.test.ts @@ -44,10 +44,6 @@ test("supported routing and cache label keys remain bounded without rewriting va { caddy_ingress_network: "dev" }, { caddy_0: "app.example" }, { "traefik.enable": "true" }, - { "com.docker.compose.project": "x" }, - { "io.hack-local.owner": "x" }, - { "io.hack-local": "x" }, - { hack: "x" }, [1], ]) { expect(hasOnlyNativeSupportedLabels(unsupported)).toBe(false); @@ -55,6 +51,11 @@ test("supported routing and cache label keys remain bounded without rewriting va // Plain metadata is admitted; the native review reports it as not applied. for (const metadata of [ { other: "value" }, + { + "com.docker.compose.project": "x", + "io.hack-local.owner": "x", + hack: "x", + }, { "com.example.team": "web" }, ["org.opencontainers.image.source=synthetic"], { hackathon: "2026" }, @@ -62,7 +63,7 @@ test("supported routing and cache label keys remain bounded without rewriting va expect(hasOnlyNativeSupportedLabels(metadata)).toBe(true); } // A mirror of the runtime's rule: the two must agree key by key. - expect(isInertMetadataLabel("com.docker.compose")).toBe(false); + expect(isInertMetadataLabel("com.docker.compose")).toBe(true); expect(isInertMetadataLabel("com.docker.composed")).toBe(true); expect( nativeBridgeCapacity({ inactive: spec, active: spec, unrouted: {} }) @@ -75,7 +76,13 @@ test("supported routing and cache label keys remain bounded without rewriting va ) ).toThrow(); }); -test("only the exact one-shot marker joins the supported label keys", () => { +test("only the exact one-shot marker joins the supported semantic label keys", () => { + expect( + hasOnlyNativeSupportedLabels({ + "io.hack-local.owner": "ignored", + "hack.service.one-shot": "true", + }) + ).toBe(true); expect( hasOnlyNativeSupportedLabels({ "hack.service.one-shot": "true" }) ).toBe(true); @@ -90,8 +97,6 @@ test("only the exact one-shot marker joins the supported label keys", () => { { "hack.service.one-shot.extra": "true" }, { "hack.service.oneshot": "true" }, { "hack.service.other": "true" }, - { "hack.x": "true" }, - { "io.hack-local.owner": "true", "hack.service.one-shot": "true" }, ["hack.service.one-shot"], ["hack.service.one-shot="], ["hack.service.one-shot=false"], From 478577b309e221803554a0ae70553bc965ef31bc Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 06:23:38 +0000 Subject: [PATCH 2/9] test(native): align ownership metadata integration contracts --- .../runtime-core/tests/project_contract.rs | 70 +++++++++++-------- 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/packages/runtime-core/tests/project_contract.rs b/packages/runtime-core/tests/project_contract.rs index 65993d8fa..88b11938e 100644 --- a/packages/runtime-core/tests/project_contract.rs +++ b/packages/runtime-core/tests/project_contract.rs @@ -608,12 +608,11 @@ fn amd64_emulation_opt_in_is_reviewed_and_never_overrides_declared_arm64() { } } -/// The candidate's own label review rejects Hack's ownership namespaces (the -/// list `HACK_OWNERSHIP_LABEL_NAMESPACES` declares in TypeScript), so a -/// project cannot carry ownership evidence even when the TypeScript classifier -/// is bypassed. Routing and dependency cache declarations never exempt them. +/// Authored ownership metadata is inert even when the TypeScript classifier is +/// bypassed. It neither blocks enrollment nor supplies ownership evidence, and +/// it does not suppress routing or dependency-cache declarations. #[test] -fn hack_ownership_label_namespaces_block_enrollment() { +fn hack_ownership_label_namespaces_are_inert_during_enrollment() { let owner_label = |fixture: &Fixture| { fixture .codes() @@ -635,25 +634,29 @@ fn hack_ownership_label_namespaces_block_enrollment() { ] { let fixture = Fixture::new(&format!("{BASIC}{labels}")); let plan = fixture.plan(); - assert!(owner_label(&fixture), "{key}: {labels}"); - assert!(!plan.plan.enrollment_compatible, "{key}"); + assert!(!owner_label(&fixture), "{key}: {labels}"); + assert!(plan.plan.enrollment_compatible, "{key}"); + assert_eq!( + plan.plan.services["web"].routing.is_some(), + labels.contains("caddy:") + ); assert!( !serde_json::to_string(&plan) .unwrap() .contains("owner-canary") ); - assert_eq!( - project::enroll(&fixture.candidate, fixture.options(), &plan.plan_id) - .unwrap_err() - .code, - "incompatible_compose" - ); assert!(!fixture.candidate.state_root.exists()); + let enrolled = + project::enroll(&fixture.candidate, fixture.options(), &plan.plan_id).unwrap(); + assert!( + !serde_json::to_string(&enrolled) + .unwrap() + .contains("owner-canary") + ); } } // The dependency cache declaration form: its own `hack.dependencies.*` - // labels are recognized, so without an ownership key nothing else reports - // an owner label. With one, the declaration does not exempt it. + // labels are recognized; ignored ownership metadata must not suppress them. let dependency_cache = |extra: &str| { Fixture::new(&format!( "services:\n deps:\n image: alpine:3.21\n volumes: ['node_modules:/app/node_modules']\n labels:\n hack.dependencies.cache-volume: node_modules\n hack.dependencies.lockfiles: bun.lock\n hack.dependencies.bootstrap: 'true'\n{extra}volumes:\n node_modules: {{}}\n" @@ -678,20 +681,21 @@ fn hack_ownership_label_namespaces_block_enrollment() { plan.plan.services["deps"].dependency_cache.is_some(), "{key}" ); - assert!(owner_label(&fixture), "{key}"); - assert!(!plan.plan.enrollment_compatible, "{key}"); + assert!(!owner_label(&fixture), "{key}"); + assert!(plan.plan.enrollment_compatible, "{key}"); assert!( !serde_json::to_string(&plan) .unwrap() .contains("owner-canary") ); - assert_eq!( - project::enroll(&fixture.candidate, fixture.options(), &plan.plan_id) - .unwrap_err() - .code, - "incompatible_compose" - ); assert!(!fixture.candidate.state_root.exists()); + let enrolled = + project::enroll(&fixture.candidate, fixture.options(), &plan.plan_id).unwrap(); + assert!( + !serde_json::to_string(&enrolled) + .unwrap() + .contains("owner-canary") + ); } // Near misses outside every ownership namespace stay reviewable. Closer // misses (a longer first segment, or another segment under the same @@ -708,8 +712,8 @@ fn hack_ownership_label_namespaces_block_enrollment() { /// Hack's documented one-shot marker (`hack.service.one-shot: "true"`, the /// label `hack config import` maps to a native job) is a completion role, not /// an ownership or router label. Only that exact key and literal value are -/// consumed; ownership namespaces next to it and every other `hack.*` key stay -/// refused, so the marker never widens the `hack.*` reservation. +/// consumed; generated ownership metadata stays inert while unsupported +/// semantic `hack.service.*` keys still refuse. #[test] fn hack_one_shot_marker_is_a_completion_role_not_an_owner_label() { const INIT: &str = "services:\n init:\n image: alpine:3.20\n command: [\"true\"]\n"; @@ -738,7 +742,7 @@ fn hack_one_shot_marker_is_a_completion_role_not_an_owner_label() { .unwrap() .contains("one_shot") ); - // Ownership namespaces next to the marker are never exempt. + // Ignored ownership metadata neither supplies evidence nor changes the role. for key in [ "com.docker.compose.project", "io.hack-local.owner", @@ -750,18 +754,22 @@ fn hack_one_shot_marker_is_a_completion_role_not_an_owner_label() { "{INIT} labels:\n hack.service.one-shot: 'true'\n '{key}': owner-canary\n" )); let plan = fixture.plan(); - assert!(owner_label(&fixture), "{key}"); - assert!(!plan.plan.enrollment_compatible, "{key}"); + assert!(!owner_label(&fixture), "{key}"); + assert!(plan.plan.enrollment_compatible, "{key}"); + assert!(plan.plan.services["init"].one_shot, "{key}"); assert!( !serde_json::to_string(&plan) .unwrap() .contains("owner-canary") ); } - // Any other `hack.*` key, including siblings of the marker, keeps the - // ownership refusal: only the exact marker key is recognized. + // Nonsemantic Hack metadata is ignored without inventing a one-shot role. + let inert = Fixture::new(&format!("{INIT} labels: {{hack.x: 'true'}}\n")); + assert!(!owner_label(&inert)); + assert!(inert.plan().plan.enrollment_compatible); + assert!(!inert.plan().plan.services["init"].one_shot); + // Unsupported semantic keys, including siblings of the marker, still refuse. for key in [ - "hack.x", "hack.service.other", "hack.service.one-shot.extra", "hack.service", From 4ebd58a84f9a9d502cf348360f1a72de6ca13d94 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 06:34:24 +0000 Subject: [PATCH 3/9] test(native): name foreign network in settings refusals --- tests/native-project-settings-cli.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/native-project-settings-cli.test.ts b/tests/native-project-settings-cli.test.ts index e43e4333a..29c4f83fc 100644 --- a/tests/native-project-settings-cli.test.ts +++ b/tests/native-project-settings-cli.test.ts @@ -196,7 +196,7 @@ test( HACK_NATIVE_ADAPTATION: explicit, }); expect(errorLines(result.output)).toEqual([ - `ERROR: E_NATIVE_PROJECT_UNSUPPORTED: Native \`hack up\` does not support an external Compose network; ${SELECTION}`, + `ERROR: E_NATIVE_PROJECT_UNSUPPORTED: Native \`hack up\` does not support an external Compose network; ${SELECTION} Foreign external networks: shared-dev.`, ]); expect(await f.calls()).toEqual([]); }, @@ -213,7 +213,7 @@ test( }); const without = await cli(f, ["up"], { HACK_NATIVE_SHARED_SOURCE: "1" }); expect(errorLines(without.output)).toEqual([ - `ERROR: E_NATIVE_PROJECT_UNSUPPORTED: Native \`hack up\` does not support an external Compose network; ${SELECTION}`, + `ERROR: E_NATIVE_PROJECT_UNSUPPORTED: Native \`hack up\` does not support an external Compose network; ${SELECTION} Foreign external networks: shared-dev.`, ]); expect(await f.calls()).toEqual([]); const withEnv = await cli(f, ["up", "--env", "qa"], { From 1f38c6ad93b0839563b290eb383d567213e1ab52 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 08:22:36 +0000 Subject: [PATCH 4/9] fix(native): infer readiness probes for standard Caddy routes --- docs/guides/native-candidate.md | 39 ++++- docs/reference/v5-model-coverage.md | 1 + packages/runtime-core/guest/http-probe.c | 47 ++++-- .../src/project/compose/service.rs | 33 ++++- .../src/provider/graph/host_relay.rs | 1 + .../runtime-core/src/provider/graph/probes.rs | 1 + .../runtime-core/src/provider/graph/routes.rs | 4 + .../runtime-core/src/provider/http_probe.rs | 138 +++++++++++++++++- .../runtime-core/tests/project_contract.rs | 91 ++++++++++++ src/backends/native-http-probe-admission.ts | 22 +++ src/backends/native-preflight.ts | 11 +- src/backends/native-project-https.ts | 97 +++++++++--- src/backends/native-project-routing.ts | 16 +- src/backends/native-project-start.ts | 72 +++++++-- tests/native-http-probe-admission.test.ts | 8 + tests/native-preflight-cli.test.ts | 26 +++- tests/native-project-https.test.ts | 58 +++++++- tests/native-project-routing.test.ts | 61 ++++++++ tests/native-project-start.test.ts | 76 ++++++++++ 19 files changed, 732 insertions(+), 70 deletions(-) diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index ee6e9b3b7..4390a5263 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -2316,10 +2316,11 @@ Host dependencies use the equivalent `--minimum-dependency-sockets N` selection; an exact capacity or a minimum. The combined physical transport limit still applies to the pool's actual capacities, including any spare slots. A minimum does not resize an existing pool or guarantee that enough slots are currently free. -Each routed service -must already declare a matching `healthcheck.x-hack-http` (see -[Native HTTP readiness probe](#native-http-readiness-probe)); command healthchecks -and missing probes are not replaced or inferred. Native review still validates +Each routed service needs a matching native route probe (see +[Native HTTP readiness probe](#native-http-readiness-probe)). If no `healthcheck` +key was authored, native infers one on the reviewed Caddy route port without +changing the Compose file. Explicit healthchecks are never replaced; command-only +or disabled checks on routed services still refuse in this increment. Native review still validates hostnames, ports, networks and labels. Foreground graph ownership supervises and retires Unix route publishers. Without explicit HTTPS selection, this enrollment does not start a hostname authority or Caddy HTTPS server. It never installs trust, configures DNS, or widens an existing pool. @@ -2838,7 +2839,35 @@ shared-source mode; active project bind mounts still use the explicit project sh #### Native HTTP readiness probe -`healthcheck.x-hack-http` declares the readiness probe native startup uses for a +For a complete Caddy route with no authored `healthcheck`, native reviews a default +probe: `GET /` on the route port, a 1000 ms interval and timeout, three consecutive +failures, and a 300000 ms startup grace matching the foreground startup deadline. +A successful TCP connection and complete HTTP/1.0 or HTTP/1.1 response headers are +required. HTTP 200–499, including `/` returning 404, is ready; 5xx, EOF, non-HTTP +bytes, a timeout or partial headers are not. Redirects are not followed. At most +four interim 1xx responses may precede the final response, within the same 8 KiB +header and timeout budgets. The request uses `Host: 127.0.0.1:`. + +This proves endpoint readiness, not application correctness. After the first +success, three consecutive failures make the service unhealthy using the existing +probe lifecycle; source-reload downtime is not silently called healthy. Final +HTTPS publication verification accepts 2xx–4xx only with a fresh challenge added +by the upstream proxy response handler. Authority denials and routing errors do +not qualify as the application's 404. No readiness decision follows an external +redirect. Explicit probes retain the stricter policy below. The generated +`policy` field is internal and refuses in authored `x-hack-http` declarations. +The native bundle embeds the matching guest probe; its content hash binds the +published helper. An older helper refuses the new policy argument. These inferred +defaults are retained-run compatibility values and cannot change without migration. + +Native reserves `X-Hack-Probe-Request` and `X-Hack-Upstream-Proof` for the HTTPS +verification exchange. The challenge is stripped before either the authority or +application sees it. Normal traffic has no challenge, so no non-empty proof +header is emitted. The shared HTTPS reader permits up to four interim 1xx +responses before the final status for both inferred and explicit routes; explicit +final-status and reviewed-redirect rules remain unchanged. + +`healthcheck.x-hack-http` declares an explicit readiness probe native startup uses for a service. It must be the only key in that service's `healthcheck`: it cannot be combined with `test`, `disable`, `interval`, `timeout`, `start_period`, `start_interval` or `retries`. Docker Compose ignores the extension field. All six diff --git a/docs/reference/v5-model-coverage.md b/docs/reference/v5-model-coverage.md index 1527be4d1..2e2a671a7 100644 --- a/docs/reference/v5-model-coverage.md +++ b/docs/reference/v5-model-coverage.md @@ -48,6 +48,7 @@ must be attached to the eventual exact integration revision. | Settled retained Compose route-attempt retirement; pinned references; interrupted deletion | **Coverage gap:** filesystem journal compaction is not modeled; native-route-stop covers a different stop protocol | Instance mutation lock and exact surviving claim witnesses in [journal compaction](../../src/lib/native-compose-route-claims.ts); [claim regressions](../../tests/native-compose-route-claims.test.ts) and [retained lifecycle integration](../../tests/native-compose-adoption-routing-generation.test.ts) | 24 successful operations keep two attempt directories and constant Linux descriptors; four integrated lifecycle operations check directory counts. Injected unlink failures cover same-store and fresh-store cleanup retries. Assumes cooperative same-user writers, the caller's mutation lock and working filesystem synchronization. No process-kill/power-loss durability proof or bound on unresolved/over-budget histories. | | Graph cleanup container stop batch: bounded in-flight stops, shared worker queue, connect-stage retry, batch-wide unreachable state, deadline expiry, worker panic | **Coverage gap:** no maintained model covers scheduling inside the stop batch; native-route-stop covers a different stop protocol | `dispatch` and `attempt` in [the stop batch](../../packages/runtime-core/src/provider/engine/stop.rs): at most 4 stops in flight, every worker joined; only `connect`/`connect_timeout` failures retry (25 ms doubling to 500 ms, within the batch deadline and 30 s of a stop's first connect failure); once one stop exhausts its window, later connect failures return at once; a deadline reached during backoff keeps the connect stage; `response`, `timeout` and `transport` never retry. Regressions in the same file's `provider::engine::stop` tests (bounded in-flight socket server, late socket, missing socket, 14-stop retry counts, persistent connect failures, oversleeping backoff, non-retried stages, worker panic) and `fourteen_service_connect_failures_keep_resumable_progress_and_retry_only_unstopped` | Tests cover exact attempt counts and the in-flight bound for fixed interleavings; no model explores all worker schedules or proves at most one delivered request per stop. Assumes a failed connect sent no request bytes and that the stop budget covers fully serialized grace periods. Graph-level resume after a partial batch is covered by tests only. Host qualification of the descriptor-pressure root cause (HACK-1302) remains open. | | Concurrent stop/publication; route claim and relay lifetime | native-route-stop, shared-https-lifetime, previous-boot-shared-https, relay-lifecycle-barrier, relay-authorization, relay-staging, relay-fence | Fenced route removal, relay owner, ready/removed protocol, authenticated observations | Native DNS/TLS/browser behavior; old-boot uncertainty and external routing. | +| Inferred Caddy route readiness without authored probes | Existing probe generation, exec identity and freshness checks are reused; no new lifecycle transition | `http_probe.rs` and compiled `guest/http-probe.c` tests cover TCP refusal, complete HTTP 2xx–4xx, 5xx, EOF, timeout, partial/oversized headers and bounded interim responses. Project contracts preserve explicit checks; TypeScript route/preflight tests require the exact reviewed inferred policy. HTTPS tests require a fresh upstream response challenge for inferred routes | 32 route slots, 1 s interval/deadline, 8 KiB headers, four interim responses, three failed samples and 300 s initial grace. No model explores HTTP parsing or Caddy handler behavior. TLS fixtures simulate the proxy marker; an actual Caddy/authority/VM run must qualify application 404 versus authority denial, SNI mismatch and missing upstream. | | Shared host helper instance: overlapping ups, session adoption, stale create decisions, replacement before finalize, epoch-record failure, interrupted cleanup, concurrent down, displaced launches, failed down session stop | shared-helper-instance | `src/lib/project-shared-helpers.ts` (shutdown generation, `assertCurrent`, `SharedHelperSessionOwnership`); `src/lib/project-lifecycle-controller.ts` session decision, the locked creation transaction (revalidate, publish, create, environment, record, stop on failure, all under one lock), launches fenced by the lock and a token check, abort and signal cleanup through the same epoch decision (`runIfOwned`), an unknown exact-presence answer failing the inspection, and the locked conditional finalize; `stopSessionBeforeReleasingHelpers` (down, restart, native retirement keep holders when the session stop fails); `runMuxCommand` bounds every lifecycle and backend mux call so it settles before the lock is released, and reports a timeout as unknown (never absence); `tests/mux-command-timeout.test.ts`; `tests/project-lifecycle-controller-ownership.test.ts`, `tests/project-shared-helpers.test.ts`, `tests/native-project-down.test.ts`, e2e `shared-host-helpers` | Helper holder/rollback and claim protocol are not modeled (claim fixtures cover them); timeout settlement is an assumption the code does not guarantee (a killed tmux client's queued command may still apply, so a late kill or window command can reach a newer session; consequences and the session-ID fence, not in this slice, are in the README); zellij is refused for shared-helper lifecycle sessions (token written after creation); the gap between a down's session stop and its release (an up that captured after the stopping mark can still succeed there) is a documented first-slice limitation; real mux/process/lock-file behavior. | | Engine restart policies: stop versus pending restart, guest engine restart, stale incarnations, terminal proof | service-restart-policy | Receipt-admitted policy, disarm-before-stop and `restarting` stop preflight in `shutdown.rs`; admission refusals for incarnation-bound delivery classes in `restart_policy.rs`; owner-driven `Rebind` after full retirement, fresh ownership/attachment checks and a new reservation in `bridges/restart_rebind.rs`; route probe sample-generation rearm; Rust tests for admission, endpoint identity, port-busy refusal, disarm and stop preflight; portable selected-child readiness guard plus macOS two-real-child polling regression prevent historical sibling exits from rejecting a replacement or misclassifying its own listener | Real guest engine behavior (crash restart, stop during backoff, disarm before deletion) needs native host evidence; several services, selected-child readiness during sibling failure, backoff timing and retry counting are not modeled. Same-port/route reachability after a policy restart, route probe rearm and per-slot child retirement still require real-VM host evidence. Rebind abstracts retirement/reservation/publication; relay lifecycle, fence and staging models cover their separate failure boundaries. | | Foreground Ctrl-C down hooks: concurrent `down`, second interrupt, crash between or during hooks, stop during `up.after`, next generation | native-foreground-down-hooks | Value-free compare-and-swap journal in [down hooks](../../src/backends/native-project-down-hooks.ts), foreground orchestration in [foreground stop](../../src/backends/native-project-foreground-stop.ts) and `hack down` in [native down](../../src/backends/native-project-down.ts); [journal](../../tests/native-project-down-hooks.test.ts), [foreground](../../tests/native-project-start-stop-hooks.test.ts) and [down](../../tests/native-project-down-journal.test.ts) regressions | Durable writes are atomic in the model; real fsync/power loss, a stale journal lock after a crash inside the lock, and the live process-group behavior of the graph owner on a host are not covered. One generation and at most two `down` invocations are explored. | diff --git a/packages/runtime-core/guest/http-probe.c b/packages/runtime-core/guest/http-probe.c index 41a994abf..25f2c4e8f 100644 --- a/packages/runtime-core/guest/http-probe.c +++ b/packages/runtime-core/guest/http-probe.c @@ -45,7 +45,7 @@ static bool wait_socket(int fd, short events, uint64_t deadline) { } return false; } -static bool request(int port, const char *path, uint64_t timeout) { +static bool request(int port, const char *path, uint64_t timeout, bool route) { uint64_t deadline = millis(CLOCK_MONOTONIC) + timeout; int fd = socket(AF_INET, SOCK_STREAM, 0); if (fd < 0) return false; @@ -71,22 +71,35 @@ static bool request(int port, const char *path, uint64_t timeout) { sent += (size_t)n; } char headers[8193]; - size_t used = 0; - while (used < sizeof(headers) - 1) { + size_t used = 0, total = 0; + unsigned interim = 0; + while (used < sizeof(headers) - 1 && total < sizeof(headers) - 1) { if (!wait_socket(fd, POLLIN, deadline)) goto done; - ssize_t n = recv(fd, headers + used, sizeof(headers) - 1 - used, 0); + ssize_t n = recv(fd, headers + used, sizeof(headers) - 1 - total, 0); if (n < 0 && (errno == EAGAIN || errno == EINTR)) continue; if (n <= 0) goto done; - size_t end = used + (size_t)n; - bool complete = false; - while (used < end) { - if (!headers[used++]) goto done; - if (used >= 4 && !memcmp(headers + used - 4, "\r\n\r\n", 4)) { complete = true; break; } - } - if (complete) { - ok = used >= 16 && (!memcmp(headers, "HTTP/1.1 ", 9) || !memcmp(headers, "HTTP/1.0 ", 9)) && - headers[9] == '2' && headers[10] >= '0' && headers[10] <= '9' && - headers[11] >= '0' && headers[11] <= '9' && headers[12] == ' '; + used += (size_t)n; + total += (size_t)n; + for (;;) { + size_t end = 0; + for (size_t i = 4; i <= used; i++) { + if (!memcmp(headers + i - 4, "\r\n\r\n", 4)) { end = i; break; } + } + if (!end) break; + if (end < 16 || memchr(headers, 0, end) || + (memcmp(headers, "HTTP/1.1 ", 9) && memcmp(headers, "HTTP/1.0 ", 9)) || + headers[9] < '1' || headers[9] > '5' || headers[10] < '0' || headers[10] > '9' || + headers[11] < '0' || headers[11] > '9' || headers[12] != ' ') goto done; + int status = (headers[9] - '0') * 100 + (headers[10] - '0') * 10 + headers[11] - '0'; + if (route && status >= 100 && status < 200 && status != 101) { + // Early hints and continue responses consume the same byte/time + // budget; they never satisfy readiness without a final status. + if (++interim > 4) goto done; + used -= end; + memmove(headers, headers + end, used); + continue; + } + ok = status >= 200 && status < (route ? 500 : 300); goto done; } } @@ -112,7 +125,9 @@ static bool publish(int directory, const char *name, const char *generation, uin return ok && !renameat(directory, "status.pending", directory, name); } int main(int argc, char **argv) { - if (argc != 9) return 125; + if (argc != 9 && argc != 10) return 125; + bool route = argc == 10; + if (route && strcmp(argv[9], "route")) return 125; int port = (int)number(argv[1], 1, 65535); size_t path_length = strlen(argv[2]); if (!path_length || path_length > 512 || argv[2][0] != '/') return 125; @@ -146,7 +161,7 @@ int main(int argc, char **argv) { if (!publish(directory, "status", argv[8], sequence, health, failures, 0)) return 125; while (!stopping) { uint64_t begun = millis(CLOCK_MONOTONIC); - bool success = request(port, argv[2], timeout); + bool success = request(port, argv[2], timeout, route); if (stopping) break; uint64_t finished = millis(CLOCK_MONOTONIC); if (success) { failures = 0; health = 1; ever_healthy = true; } diff --git a/packages/runtime-core/src/project/compose/service.rs b/packages/runtime-core/src/project/compose/service.rs index b574358e5..c98d4ec36 100644 --- a/packages/runtime-core/src/project/compose/service.rs +++ b/packages/runtime-core/src/project/compose/service.rs @@ -390,6 +390,12 @@ fn health( problem("invalid_healthcheck", "Invalid native HTTP declaration.") })?; probe.validate()?; + if probe.policy.is_some() { + return Err(problem( + "invalid_healthcheck", + "Native HTTP policy is generated from routing, not authored in a healthcheck.", + )); + } if m.keys().any(|key| key != "x-hack-http") { return Err(problem( "invalid_healthcheck", @@ -1101,6 +1107,31 @@ pub(super) fn compile( "Dependency cache volume must be mounted by its declaring service; values omitted.", )); } + let mut healthcheck = health( + m.get("healthcheck"), + &format!("{field}.healthcheck"), + diagnostics, + )?; + if !m.contains_key("healthcheck") { + if let Some(route) = &routing { + healthcheck = Some(HealthPlan { + native_http: Some(crate::provider::http_probe::HttpProbe::inferred_route( + route.port, + )), + disabled: false, + test: None, + interval_nanos: None, + timeout_nanos: None, + start_period_nanos: None, + start_interval_nanos: None, + retries: None, + }); + diagnostics.push(Diagnostic::warning( + "inferred_route_probe", &format!("{field}.healthcheck"), + "Native infers a route-port probe requiring TCP and complete HTTP 200-499 response headers. HTTP 5xx, EOF, timeouts and non-HTTP responses are not ready. No explicit healthcheck was changed.", + )); + } + } result.insert(name.clone(),ServicePlan { image,build, command:command(m.get("command"),&format!("{field}.command"))?, @@ -1111,7 +1142,7 @@ pub(super) fn compile( environment:redacted_map(m.get("environment"),true)?,extra_hosts, environment_files:environment_files(project,base,m.get("env_file"),&format!("{field}.env_file"),diagnostics)?, environment_precedence:"Compose environment keys override explicitly listed env_file inputs; bare keys and references remain unresolved; no ambient/.env values are loaded".into(), - healthcheck:health(m.get("healthcheck"),&format!("{field}.healthcheck"),diagnostics)?, + healthcheck, ports:ports(m.get("ports"),&format!("{field}.ports"),diagnostics)?,exposed_ports,networks,network_aliases,network_mode:if ["project-networks", "none"].contains(&network_mode) {network_mode.into()} else {"[unsupported]".into()}, limits:limits(m,&field,diagnostics)?,read_only:boolean(m.get("read_only"),false)?,init:boolean(m.get("init"),false)?,restart, working_dir:m.get("working_dir").map(|v| safe_absolute(string(v)?)).transpose()?, diff --git a/packages/runtime-core/src/provider/graph/host_relay.rs b/packages/runtime-core/src/provider/graph/host_relay.rs index 0786c8464..590d8353e 100644 --- a/packages/runtime-core/src/provider/graph/host_relay.rs +++ b/packages/runtime-core/src/provider/graph/host_relay.rs @@ -1151,6 +1151,7 @@ mod tests { timeout_ms: 100, retries: 1, start_period_ms: 0, + policy: None, }, allocation: "1".repeat(32), generation: "2".repeat(32), diff --git a/packages/runtime-core/src/provider/graph/probes.rs b/packages/runtime-core/src/provider/graph/probes.rs index 65eb0da7d..7fc28b88f 100644 --- a/packages/runtime-core/src/provider/graph/probes.rs +++ b/packages/runtime-core/src/provider/graph/probes.rs @@ -280,6 +280,7 @@ mod observation_clock_tests { timeout_ms: 40, retries: 3, start_period_ms: 0, + policy: None, }, allocation: "a".repeat(32), generation: "b".repeat(32), diff --git a/packages/runtime-core/src/provider/graph/routes.rs b/packages/runtime-core/src/provider/graph/routes.rs index e73a651de..503339d7c 100644 --- a/packages/runtime-core/src/provider/graph/routes.rs +++ b/packages/runtime-core/src/provider/graph/routes.rs @@ -56,6 +56,10 @@ fn validate_services( || service.network_mode != "project-networks" || service.networks.is_empty() || !probe.is_some_and(|probe| probe.port == route.port && probe.validate().is_ok()) + || probe.is_some_and(|probe| { + probe.policy.is_some() + && *probe != super::super::http_probe::HttpProbe::inferred_route(route.port) + }) || !valid_intent(route) || !["caddy", "caddy.reverse_proxy", "caddy.tls"] .iter() diff --git a/packages/runtime-core/src/provider/http_probe.rs b/packages/runtime-core/src/provider/http_probe.rs index 75d422c0d..76534a784 100644 --- a/packages/runtime-core/src/provider/http_probe.rs +++ b/packages/runtime-core/src/provider/http_probe.rs @@ -12,6 +12,14 @@ pub struct HttpProbe { pub timeout_ms: u64, pub retries: u32, pub start_period_ms: u64, + /// Generated route readiness only; absent for authored HTTP healthchecks. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub policy: Option, +} +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum HttpProbePolicy { + Route, } fn invalid() -> CandidateError { CandidateError::new( @@ -26,6 +34,21 @@ fn generation_valid(value: &str) -> bool { .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) } impl HttpProbe { + /// A reviewed route without an authored healthcheck gets a bounded localhost + /// TCP connection followed by a complete HTTP response from that endpoint. + pub fn inferred_route(port: u16) -> Self { + // These defaults are part of retained route admission in Rust and + // TypeScript. Changing them requires a compatibility migration. + Self { + port, + path: "/".into(), + interval_ms: 1_000, + timeout_ms: 1_000, + retries: 3, + start_period_ms: 300_000, + policy: Some(HttpProbePolicy::Route), + } + } pub fn validate(&self) -> Result<(), CandidateError> { if self.port == 0 || !self.path.starts_with('/') @@ -56,7 +79,7 @@ impl HttpProbe { { return Err(invalid()); } - Ok(vec![ + let mut arguments = vec![ self.port.to_string(), self.path.clone(), self.interval_ms.to_string(), @@ -65,7 +88,11 @@ impl HttpProbe { self.start_period_ms.to_string(), directory.into(), generation.into(), - ]) + ]; + if self.policy == Some(HttpProbePolicy::Route) { + arguments.push("route".into()); + } + Ok(arguments) } /// A historical healthy sample cannot outlive its exec, generation, or freshness window. pub fn health( @@ -125,6 +152,8 @@ impl HttpProbe { } #[cfg(feature = "native-http-probe")] pub fn guest_binary() -> &'static [u8] { + // Built from this crate's guest source by build.rs and embedded in the same + // runtime as the planner; publication addresses it by its content hash. include_bytes!(concat!(env!("OUT_DIR"), "/http-probe")) } @@ -140,6 +169,7 @@ mod tests { timeout_ms: 40, retries: 3, start_period_ms: 0, + policy: None, } } #[test] @@ -202,6 +232,38 @@ mod tests { assert!(p.validate().is_err()); } + #[test] + fn inferred_policy_is_reviewed_while_explicit_serialization_and_arguments_stay_unchanged() { + let explicit = config(); + assert!( + serde_json::to_value(&explicit) + .unwrap() + .get("policy") + .is_none() + ); + assert_eq!( + explicit + .arguments("/private/probe", GENERATION) + .unwrap() + .len(), + 8 + ); + let inferred = HttpProbe::inferred_route(3000); + inferred.validate().unwrap(); + assert_eq!(serde_json::to_value(&inferred).unwrap()["policy"], "route"); + assert_eq!( + inferred + .arguments("/private/probe", GENERATION) + .unwrap() + .last() + .unwrap(), + "route" + ); + let mut invalid = serde_json::to_value(&inferred).unwrap(); + invalid["policy"] = serde_json::json!("anything"); + assert!(serde_json::from_value::(invalid).is_err()); + } + #[cfg(feature = "native-http-probe")] mod native { use super::*; @@ -235,6 +297,14 @@ mod tests { Self::with_timeout(mode, grace, 40) } fn with_timeout(mode: usize, grace: u64, timeout: u64) -> Self { + Self::with_policy(mode, grace, timeout, None) + } + fn with_policy( + mode: usize, + grace: u64, + timeout: u64, + policy: Option, + ) -> Self { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let port = listener.local_addr().unwrap().port(); listener.set_nonblocking(true).unwrap(); @@ -274,6 +344,19 @@ mod tests { } continue; } + 7 => b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n", + 8 => b"HTTP/1.1 499 Client Closed\r\n\r\n", + 9 => b"HTTP/1.1 500 Failed\r\n\r\n", + 10 => b"HTTP/", + 11 => b"", + 12 => b"HTTP/1.1 103 Early Hints\r\n\r\nHTTP/1.1 200 OK\r\n\r\n", + 13 => b"HTTP/1.1 103 Early Hints\r\n\r\nHTTP/1.1 503 Failed\r\n\r\n", + 14 => b"HTTP/1.1 103 Early Hints\r\n\r\nHTTP/1.1 103 Early Hints\r\n\r\nHTTP/1.1 103 Early Hints\r\n\r\nHTTP/1.1 103 Early Hints\r\n\r\nHTTP/1.1 103 Early Hints\r\n\r\nHTTP/1.1 200 OK\r\n\r\n", + 15 => { + let response = format!("HTTP/1.1 200 OK\r\nX-Large: {}\r\n\r\n", "x".repeat(8192)); + let _ = stream.write_all(response.as_bytes()); + continue; + } _ => b"NOT-HTTP 200 OK\r\n\r\n", }; let _ = stream.write_all(response); @@ -294,6 +377,10 @@ mod tests { config.port = port; config.start_period_ms = grace; config.timeout_ms = timeout; + config.policy = policy; + if policy == Some(HttpProbePolicy::Route) { + config.path = "/".into(); + } let mut fixture = Self { root, process: None, @@ -358,7 +445,9 @@ mod tests { let _ = p.wait(); } self.stop.store(1, Ordering::SeqCst); - self.server.take().unwrap().join().unwrap(); + if let Some(server) = self.server.take() { + server.join().unwrap(); + } fs::remove_dir_all(&self.root).unwrap(); } } @@ -474,6 +563,49 @@ mod tests { ); } #[test] + fn inferred_route_accepts_http_2xx_through_4xx_but_never_tcp_only_or_http_5xx() { + for (mode, expected) in [ + (1, Health::Healthy), + (2, Health::Unhealthy), // TCP without HTTP is not a ready route. + (3, Health::Healthy), // Redirect is observed, never followed. + (5, Health::Unhealthy), + (7, Health::Healthy), + (8, Health::Healthy), + (11, Health::Unhealthy), + (0, Health::Unhealthy), + (4, Health::Unhealthy), + (6, Health::Unhealthy), // Slowly delivered partial HTTP cannot fall back to TCP. + (9, Health::Unhealthy), + (10, Health::Unhealthy), + (12, Health::Healthy), + (13, Health::Unhealthy), + (14, Health::Unhealthy), + (15, Health::Unhealthy), + ] { + let mut f = Fixture::with_policy(mode, 0, 40, Some(HttpProbePolicy::Route)); + f.wait(expected, 0); + assert!(f.accepted.load(Ordering::SeqCst) > 0); + } + for mode in [2, 3, 5, 7, 8, 11] { + let mut f = Fixture::new(mode, 0); + f.wait(Health::Unhealthy, 0); + } + } + + #[test] + fn inferred_route_requires_tcp_and_later_5xx_revokes_readiness() { + let mut f = Fixture::with_policy(7, 0, 40, Some(HttpProbePolicy::Route)); + let ready = f.wait(Health::Healthy, 0); + f.mode.store(9, Ordering::SeqCst); + let down = f.wait(Health::Unhealthy, ready); + assert!(down >= ready + 3); + f.mode.store(7, Ordering::SeqCst); + let ready = f.wait(Health::Healthy, down); + f.stop.store(1, Ordering::SeqCst); + f.server.take().unwrap().join().unwrap(); + f.wait(Health::Unhealthy, ready); + } + #[test] fn first_sample_survives_recovery_without_overriding_live_health() { for mode in [0, 1] { let mut f = Fixture::new(mode, 3000); diff --git a/packages/runtime-core/tests/project_contract.rs b/packages/runtime-core/tests/project_contract.rs index 88b11938e..b35180501 100644 --- a/packages/runtime-core/tests/project_contract.rs +++ b/packages/runtime-core/tests/project_contract.rs @@ -66,6 +66,97 @@ impl Drop for Fixture { } const BASIC: &str = "services:\n web:\n image: alpine:3.21\n command: [echo, hello]\n"; +#[test] +fn standard_route_without_healthcheck_infers_a_reviewed_probe_without_editing_source() { + use hack_runtime_core::provider::http_probe::HttpProbe; + let document = format!( + "{BASIC} labels:\n caddy: web.example.hack\n caddy.reverse_proxy: '{{{{upstreams 3000}}}}'\n caddy.tls: internal\n" + ); + let fixture = Fixture::new(&document); + let report = fixture.plan(); + assert!(report.plan.enrollment_compatible); + let health = report.plan.services["web"].healthcheck.as_ref().unwrap(); + assert!(!health.disabled); + assert!(health.test.is_none()); + assert_eq!(health.native_http, Some(HttpProbe::inferred_route(3000))); + assert!( + report + .plan + .diagnostics + .iter() + .any(|d| d.code == "inferred_route_probe" && d.severity == "warning") + ); + assert_eq!( + fs::read_to_string(fixture.project.join("compose.yaml")).unwrap(), + document + ); + assert!(!fixture.candidate.state_root.exists()); +} + +#[test] +fn route_inference_never_replaces_authored_healthchecks_or_accepts_an_authored_policy() { + let route = format!( + "{BASIC} labels:\n caddy: web.example.hack\n caddy.reverse_proxy: '{{{{upstreams 3000}}}}'\n caddy.tls: internal\n" + ); + for authored in [ + "{}", + "{disable: true}", + "{test: [CMD, 'true']}", + "{test: [NONE]}", + ] { + let fixture = Fixture::new(&format!("{route} healthcheck: {authored}\n")); + let report = fixture.plan(); + assert!( + report.plan.services["web"] + .healthcheck + .as_ref() + .unwrap() + .native_http + .is_none(), + "{authored}" + ); + assert!( + !report + .plan + .diagnostics + .iter() + .any(|d| d.code == "inferred_route_probe") + ); + } + let probe = "{port: 3000, path: /health, interval_ms: 100, timeout_ms: 500, retries: 3, start_period_ms: 0}"; + let fixture = Fixture::new(&format!( + "{route} healthcheck: {{x-hack-http: {probe}}}\n" + )); + let report = fixture.plan(); + let health = report.plan.services["web"] + .healthcheck + .as_ref() + .unwrap() + .native_http + .as_ref() + .unwrap(); + assert_eq!(health.path, "/health"); + assert!(health.policy.is_none()); + assert!( + serde_json::to_value(health) + .unwrap() + .get("policy") + .is_none() + ); + let forged = probe.replace("path: /health", "path: /, policy: route"); + fs::write( + fixture.project.join("compose.yaml"), + format!("{route} healthcheck: {{x-hack-http: {forged}}}\n"), + ) + .unwrap(); + assert_eq!( + project::plan(&fixture.candidate, fixture.options()) + .unwrap_err() + .code, + "invalid_healthcheck" + ); +} + #[test] fn local_https_labels_have_typed_review_intent_without_publication() { let labels = " labels:\n caddy: 'search.example-app.hack, search.example-app.hack.gy'\n caddy.reverse_proxy: '{{upstreams 6980}}'\n caddy.tls: internal\n"; diff --git a/src/backends/native-http-probe-admission.ts b/src/backends/native-http-probe-admission.ts index 10ff6cb07..a277009df 100644 --- a/src/backends/native-http-probe-admission.ts +++ b/src/backends/native-http-probe-admission.ts @@ -11,6 +11,28 @@ const FIELDS = new Set([ "start_period_ms", ]); const PATH = /^\/[\x21-\x22\x24-\x7e]*$/; + +/** + * Exact compiler-generated defaults; this policy is never an authored opt-in. + * These values bind retained route admission; changing them requires migration. + */ +export function isInferredNativeRouteProbe( + probe: unknown, + port: unknown +): boolean { + return ( + isRecord(probe) && + bounded(port, 1, 65_535) && + Object.keys(probe).length === 7 && + probe.policy === "route" && + probe.port === port && + probe.path === "/" && + probe.interval_ms === 1000 && + probe.timeout_ms === 1000 && + probe.retries === 3 && + probe.start_period_ms === 300_000 + ); +} function bounded(value: unknown, min: number, max: number): boolean { return ( typeof value === "number" && diff --git a/src/backends/native-preflight.ts b/src/backends/native-preflight.ts index ffba7db22..2c226d0b5 100644 --- a/src/backends/native-preflight.ts +++ b/src/backends/native-preflight.ts @@ -425,7 +425,12 @@ const RUNTIME_FINDINGS: Readonly> = { route_binding_required: { message: "The service declares a route, which needs native HTTPS to be set up.", - fix: "Set up native HTTPS (see the guide) and give the service a healthcheck with x-hack-http.", + fix: "Set up native HTTPS (see the guide). A route without an authored healthcheck gets a default route-port probe.", + }, + inferred_route_probe: { + message: + "Native checks TCP and HTTP on the route port because no healthcheck was authored. HTTP 2xx-4xx counts as ready; 5xx does not.", + fix: "Nothing to do. Declare an explicit x-hack-http healthcheck if this route needs a different path or timing.", }, inert_route_declaration: { message: @@ -1328,8 +1333,8 @@ function routeFindings(opts: { }, { message: - "The service has caddy routing labels but no enabled native HTTP probe on its route port, or it also declares a dependency cache.", - fix: "Give the service a healthcheck with x-hack-http on the port its route proxies to, or add one under httpProbes in the native adaptation.", + "The route lacks matching reviewed readiness, has an incompatible explicit healthcheck, or also declares a dependency cache.", + fix: "Keep the default inferred route probe, or give an explicit healthcheck x-hack-http on the route port. Disabled or command-only healthchecks and dependency-cache routes still refuse.", } ) ); diff --git a/src/backends/native-project-https.ts b/src/backends/native-project-https.ts index 0a924f516..4a51c4b71 100644 --- a/src/backends/native-project-https.ts +++ b/src/backends/native-project-https.ts @@ -38,7 +38,7 @@ import { } from "./native-runtime-client.ts"; const PROBE_PATH_BYTES = /^[\x21-\x7e]+$/; -const HTTP_STATUS = /^HTTP\/1\.[01] ([2-5][0-9]{2})(?: [^\r\n]*)?$/; +const HTTP_STATUS = /^HTTP\/1\.[01] ([1-5][0-9]{2})(?: [^\r\n]*)?$/; const SHA = /^[a-f0-9]{64}$/; const SINGLE_CERTIFICATE = /^-----BEGIN CERTIFICATE-----\r?\n[A-Za-z0-9+/=\r\n]+-----END CERTIFICATE-----$/; @@ -348,6 +348,8 @@ https://:${opts.httpsPort} { on_demand } route { + vars native_probe {http.request.header.X-Hack-Probe-Request} + request_header -X-Hack-Probe-Request request_header -X-Hack-Endpoint forward_auth ${quoted(`unix/${opts.authority}`)} { uri /route? @@ -356,6 +358,7 @@ https://:${opts.httpsPort} { reverse_proxy unix/{http.request.header.X-Hack-Endpoint} { header_up Host {hostport} header_up -X-Hack-Endpoint + header_down X-Hack-Upstream-Proof {http.vars.native_probe} } } } @@ -1115,6 +1118,8 @@ export function isNativeHttpsProbePath(path: unknown): path is string { export interface NativeHttpsResponse { readonly statusCode: number; readonly location?: string; + /** A fresh request challenge was added by the upstream proxy response handler. */ + readonly upstreamVerified?: true; } async function confirmTlsPeer(opts: { readonly verifyPeer?: (peerPort: number) => Promise; @@ -1130,7 +1135,27 @@ async function confirmTlsPeer(opts: { } const HEADER_NAME = /^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/; const HEADER_VALUE = /^[\x20-\x7e\t]*$/; -export function parseNativeHttpsHeaders(header: string): NativeHttpsResponse { +function parseHttpsHeaderField(line: string): { name: string; value: string } { + const colon = line.indexOf(":"); + const name = line.slice(0, colon).toLowerCase(); + const raw = line.slice(colon + 1); + if (colon < 1 || !HEADER_NAME.test(name) || !HEADER_VALUE.test(raw)) { + throw refused(); + } + return { name, value: raw.trim() }; +} +function confirmedUpstream( + expectedProof: string | undefined, + observedProof: string | undefined +): Pick { + return expectedProof !== undefined && observedProof === expectedProof + ? { upstreamVerified: true } + : {}; +} +export function parseNativeHttpsHeaders( + header: string, + expectedProof?: string +): NativeHttpsResponse { if (header.length > 8192 || !header.endsWith("\r\n\r\n")) { throw refused(); } @@ -1141,27 +1166,60 @@ export function parseNativeHttpsHeaders(header: string): NativeHttpsResponse { throw refused(); } let location: string | undefined; + let upstreamProof: string | undefined; for (const line of lines) { - const colon = line.indexOf(":"); - const name = line.slice(0, colon).toLowerCase(); - const value = line.slice(colon + 1).trim(); - if ( - colon < 1 || - !HEADER_NAME.test(name) || - !HEADER_VALUE.test(line.slice(colon + 1)) - ) { - throw refused(); - } + const { name, value } = parseHttpsHeaderField(line); if (name === "location") { if (location !== undefined || !value || value.length > 2048) { throw refused(); } location = value; } + if (name === "x-hack-upstream-proof") { + if (upstreamProof !== undefined) { + throw refused(); + } + upstreamProof = value; + } } return { statusCode: Number(match[1]), ...(location === undefined ? {} : { location }), + ...confirmedUpstream(expectedProof, upstreamProof), + }; +} +/** One byte budget spans interim and final headers; bodies are never retained. */ +function httpsResponseReader( + upstreamProof: string +): (data: Buffer) => NativeHttpsResponse | undefined { + let prefix = ""; + let headerBytes = 0; + let interim = 0; + return (data) => { + const part = data.subarray(0, 8192 - headerBytes); + headerBytes += part.length; + prefix += part.toString("latin1"); + while (true) { + const end = prefix.indexOf("\r\n\r\n"); + if (end < 0) { + if (headerBytes >= 8192) { + throw refused(); + } + return undefined; + } + const response = parseNativeHttpsHeaders( + prefix.slice(0, end + 4), + upstreamProof + ); + if (response.statusCode >= 200) { + return response; + } + interim++; + if (response.statusCode === 101 || interim > 4) { + throw refused(); + } + prefix = prefix.slice(end + 4); + } }; } export async function verifyNativeHttpsHostname( @@ -1182,9 +1240,10 @@ export async function verifyNativeHttpsHostname( throw refused(); } const ca = await validCa(caPath); + const upstreamProof = randomBytes(16).toString("hex"); return await new Promise((resolve, reject) => { let settled = false; - let prefix = ""; + const readResponse = httpsResponseReader(upstreamProof); let phase: "handshake" | "response" = "handshake"; let peerVerified = verifyPeer === undefined; const finish = (error?: Error, response?: NativeHttpsResponse) => { @@ -1223,7 +1282,7 @@ export async function verifyNativeHttpsHostname( phase = "response"; const authority = port === 443 ? hostname : `${hostname}:${port}`; socket.write( - `GET ${path} HTTP/1.1\r\nHost: ${authority}\r\nConnection: close\r\n\r\n` + `GET ${path} HTTP/1.1\r\nHost: ${authority}\r\nX-Hack-Probe-Request: ${upstreamProof}\r\nConnection: close\r\n\r\n` ); } catch { finish(nativeHttpsVerificationError(undefined)); @@ -1236,13 +1295,11 @@ export async function verifyNativeHttpsHostname( return; } // Parse only bounded headers; discard any body bytes delivered in the same chunk. - prefix += data.subarray(0, 8192 - prefix.length).toString("latin1"); - const end = prefix.indexOf("\r\n\r\n"); - if (end < 0 && prefix.length < 8192) { - return; - } try { - finish(undefined, parseNativeHttpsHeaders(prefix.slice(0, end + 4))); + const response = readResponse(data); + if (response !== undefined) { + finish(undefined, response); + } } catch { finish(nativeHttpsVerificationError(undefined)); } diff --git a/src/backends/native-project-routing.ts b/src/backends/native-project-routing.ts index f31e38022..586095ba4 100644 --- a/src/backends/native-project-routing.ts +++ b/src/backends/native-project-routing.ts @@ -2,6 +2,7 @@ import { HackCliError } from "../lib/cli-result.ts"; import { isRecord } from "../lib/guards.ts"; import { HACK_OWNERSHIP_LABEL_NAMESPACES } from "../lib/hack-ownership-labels.ts"; import { isIgnoredHackComposeLabel } from "../lib/native-hack-compose-conventions.ts"; +import { isInferredNativeRouteProbe } from "./native-http-probe-admission.ts"; import { hasOnlyNativeCacheLabels } from "./native-project-cache.ts"; const ROUTE_LABELS = new Set(["caddy", "caddy.reverse_proxy", "caddy.tls"]); @@ -10,7 +11,7 @@ function refused(): Error { return new HackCliError({ code: "E_NATIVE_PROJECT_UNSUPPORTED", message: - "Native routes require reviewed Caddy labels and existing matching native HTTP health probes; no probe or route was invented.", + "Native routes require reviewed Caddy labels and matching native route readiness; explicit healthchecks are never replaced.", }); } /** @@ -148,16 +149,25 @@ function requireRouteProbe( const reviewedProbe = isRecord(service.healthcheck) ? service.healthcheck.native_http : undefined; + const inferred = + spec !== undefined && + !Object.hasOwn(spec, "healthcheck") && + isRecord(service.routing) && + isInferredNativeRouteProbe(reviewedProbe, service.routing.port); if ( !( isRecord(service.routing) && - isRecord(probe) && isRecord(reviewedProbe) && Number.isInteger(service.routing.port) ) || Number(service.routing.port) < 1 || Number(service.routing.port) > 65_535 || - probe.port !== service.routing.port || + !( + inferred || + (isRecord(probe) && + probe.port === service.routing.port && + reviewedProbe.policy === undefined) + ) || reviewedProbe.port !== service.routing.port || (isRecord(service.healthcheck) && service.healthcheck.disabled === true) || (service.dependency_cache !== undefined && diff --git a/src/backends/native-project-start.ts b/src/backends/native-project-start.ts index fb8f6998d..81cc7a3fe 100644 --- a/src/backends/native-project-start.ts +++ b/src/backends/native-project-start.ts @@ -24,7 +24,10 @@ import { adaptNativeGcloudEnvironment, type NativeGcloudSelection, } from "./native-gcloud-environment.ts"; -import { admitNativeHttpProbe } from "./native-http-probe-admission.ts"; +import { + admitNativeHttpProbe, + isInferredNativeRouteProbe, +} from "./native-http-probe-admission.ts"; import { acquireNativeHttpsLease, NativeHttpsLeaseAcquisitionError, @@ -288,14 +291,44 @@ async function acquireReadyHttps(opts: { * routed service without an enabled reviewed HTTP probe, or one hostname with * conflicting probes, is the configuration refusal. */ +function reviewedHttpsProbe( + healthcheck: unknown, + port: unknown +): { path: string; inferred?: true } { + if ( + !( + isRecord(healthcheck) && + healthcheck.disabled !== true && + isRecord(healthcheck.native_http) && + isNativeHttpsProbePath(healthcheck.native_http.path) + ) + ) { + throw refused(); + } + const probe = healthcheck.native_http; + const path = healthcheck.native_http.path; + const inferred = isInferredNativeRouteProbe(probe, port); + if (probe.policy !== undefined && !inferred) { + throw refused(); + } + return { path, ...(inferred ? { inferred: true as const } : {}) }; +} export function reviewedNativeHttpsRoutes( plan: unknown, services: ReadonlySet -): readonly { hostname: string; path: string; service: string }[] { +): readonly { + hostname: string; + path: string; + service: string; + inferred?: true; +}[] { if (!(isRecord(plan) && isRecord(plan.services))) { throw unconfirmed("the reviewed runtime plan"); } - const names = new Map(); + const names = new Map< + string, + { path: string; service: string; inferred?: true } + >(); for (const name of services) { const service = plan.services[name]; if ( @@ -307,21 +340,12 @@ export function reviewedNativeHttpsRoutes( ) { throw unconfirmed("the reviewed runtime plan"); } - if ( - !( - isRecord(service.healthcheck) && - service.healthcheck.disabled !== true && - isRecord(service.healthcheck.native_http) && - isNativeHttpsProbePath(service.healthcheck.native_http.path) - ) - ) { - throw refused(); - } + const probe = reviewedHttpsProbe(service.healthcheck, service.routing.port); for (const hostname of service.routing.hostnames) { if (typeof hostname !== "string") { throw unconfirmed("the reviewed runtime plan"); } - const path = service.healthcheck.native_http.path; + const path = probe.path; if ( names.has(hostname) && (names.get(hostname)?.path !== path || @@ -329,7 +353,11 @@ export function reviewedNativeHttpsRoutes( ) { throw refused(); } - names.set(hostname, { path, service: name }); + names.set(hostname, { + path, + service: name, + ...(probe.inferred ? { inferred: true as const } : {}), + }); } } return [...names] @@ -843,6 +871,18 @@ function requireEnrollmentCompatible(plan: unknown): void { } const HTTPS_REDIRECTS = new Set([301, 302, 303, 307, 308]); type HttpsRoute = ReturnType[number]; +function httpsResponseReady( + route: HttpsRoute, + response: { statusCode: number; upstreamVerified?: true } +): boolean { + if (route.inferred && response.upstreamVerified !== true) { + throw httpsVerificationFailed("UPSTREAM_RESPONSE_UNCONFIRMED"); + } + return ( + response.statusCode >= 200 && + response.statusCode < (route.inferred ? 500 : 300) + ); +} function redirectHostname( location: string, hostname: string, @@ -909,7 +949,7 @@ export async function verifyHttpsRoutes( if (!(response && Number.isInteger(response.statusCode))) { throw httpsVerificationFailed("NO_RESPONSE"); } - if (response.statusCode >= 200 && response.statusCode < 300) { + if (httpsResponseReady(route, response)) { break; } if (!(HTTPS_REDIRECTS.has(response.statusCode) && response.location)) { diff --git a/tests/native-http-probe-admission.test.ts b/tests/native-http-probe-admission.test.ts index 388c1c6ce..8a811c3b4 100644 --- a/tests/native-http-probe-admission.test.ts +++ b/tests/native-http-probe-admission.test.ts @@ -55,3 +55,11 @@ test("HTTP preflight accepts runtime boundary values and command probes", () => admitNativeHttpProbe("web", { healthcheck: { test: ["CMD", "true"] } }) ).not.toThrow(); }); + +test("an authored probe cannot opt into the generated route policy", () => { + expect(() => + admitNativeHttpProbe("web", { + healthcheck: { "x-hack-http": { ...valid, policy: "route" } }, + }) + ).toThrow("healthcheck.x-hack-http"); +}); diff --git a/tests/native-preflight-cli.test.ts b/tests/native-preflight-cli.test.ts index dde2d5d60..bb3b5b758 100644 --- a/tests/native-preflight-cli.test.ts +++ b/tests/native-preflight-cli.test.ts @@ -716,7 +716,7 @@ test( test( "route admission, project-wide hosts and per-environment dependency files are checked as native up checks them", async () => { - // A routed service without a native HTTP probe: the review only warns, native up refuses. + // A route missing its reviewed probe still refuses; authored labels alone are insufficient. const routed = await fixture({ services: { web: { @@ -743,6 +743,30 @@ test( expect(route.code).toBe(1); expect(route.report.ready).toBe(false); expect(codes(route.report, "cli")).toEqual(["route_probe_required:web"]); + await routed.review([], { + services: { + web: { + active: true, + routing: { port: 3000 }, + healthcheck: { + disabled: false, + native_http: { + policy: "route", + port: 3000, + path: "/", + interval_ms: 1000, + timeout_ms: 1000, + retries: 3, + start_period_ms: 300_000, + }, + }, + }, + }, + }); + const inferred = await preflight(routed, { ...routed.native, ...GRANT }); + expect(codes(inferred.report, "cli")).toEqual([]); + expect(inferred.code).toBe(0); + expect(inferred.report.ready).toBe(true); // Routes plus published ports over the pool's bridge limit: native up // refuses before any runtime work, so no review is needed to see it. diff --git a/tests/native-project-https.test.ts b/tests/native-project-https.test.ts index 58d05b5f1..46fdf0db5 100644 --- a/tests/native-project-https.test.ts +++ b/tests/native-project-https.test.ts @@ -245,6 +245,13 @@ test("owns authority and Caddy lifetimes, writes strict routing config and prese expect(f.config()).toContain("uri /route?"); expect(f.config()).toContain("request_header -X-Hack-Endpoint"); expect(f.config()).toContain("header_up -X-Hack-Endpoint"); + expect(f.config()).toContain( + "vars native_probe {http.request.header.X-Hack-Probe-Request}" + ); + expect(f.config()).toContain("request_header -X-Hack-Probe-Request"); + expect(f.config()).toContain( + "header_down X-Hack-Upstream-Proof {http.vars.native_probe}" + ); expect(f.config()).toContain("https://:443"); expect(f.config()).toContain("bind 127.0.0.1"); expect(f.config()).toContain("permission http http://127.0.0.1:18443/ask"); @@ -719,7 +726,11 @@ while True: if data: seen.update(request=data.decode(),remote=remote[0],remote_port=remote[1]) (root/'seen').write_text(json.dumps(seen)) - conn.sendall((root/'response').read_bytes()) + response=(root/'response').read_bytes() + if (root/'echo-proof').exists(): + proof=next((line.split(b': ',1)[1] for line in data.split(b'\r\n') if line.startswith(b'X-Hack-Probe-Request: ')),b'') + response=response.replace(b'\r\n\r\n',b'\r\nX-Hack-Upstream-Proof: '+proof+b'\r\n\r\n') + conn.sendall(response) except (OSError,ssl.SSLError): client.close() ` ); @@ -747,7 +758,11 @@ while True: ).toEqual({ statusCode: 204 }); expect(JSON.parse(await readFile(seenPath, "utf8"))).toMatchObject({ sni: "fixture.invalid", - request: `GET /health HTTP/1.1\r\nHost: fixture.invalid:${address.port}\r\nConnection: close\r\n\r\n`, + request: expect.stringMatching( + new RegExp( + `^GET /health HTTP/1\\.1\\r\\nHost: fixture\\.invalid:${address.port}\\r\\nX-Hack-Probe-Request: [a-f0-9]{32}\\r\\nConnection: close\\r\\n\\r\\n$` + ) + ), remote: "127.0.0.1", }); await rm(seenPath); @@ -809,6 +824,19 @@ while True: "/health" ) ).toEqual({ statusCode: 307, location: "https://canonical.invalid/health" }); + await writeFile(join(root, "echo-proof"), ""); + await writeFile(responsePath, "HTTP/1.1 404 Not Found\r\n\r\n"); + expect( + await verifyNativeHttpsHostname("fixture.invalid", address.port, cert, "/") + ).toEqual({ statusCode: 404, upstreamVerified: true }); + await writeFile( + responsePath, + "HTTP/1.1 103 Early Hints\r\n\r\nHTTP/1.1 404 Not Found\r\n\r\n" + ); + expect( + await verifyNativeHttpsHostname("fixture.invalid", address.port, cert, "/") + ).toEqual({ statusCode: 404, upstreamVerified: true }); + await rm(join(root, "echo-proof")); for (const invalid of [ "HTTP/1.1 100 Continue\r\n\r\n", "malformed\r\n", @@ -861,6 +889,32 @@ test("HTTPS header parsing refuses duplicate, folded and oversized fields", () = } }); +test("upstream proof requires this request's exact challenge, never merely a 4xx status", () => { + const proof = "a".repeat(32); + for (const response of [ + "HTTP/1.1 403 Denied\r\n\r\n", + "HTTP/1.1 404 Missing\r\nX-Hack-Upstream-Proof: stale\r\n\r\n", + "HTTP/1.1 421 Misdirected\r\n\r\n", + "HTTP/1.1 502 Missing Socket\r\n\r\n", + ]) { + expect( + parseNativeHttpsHeaders(response, proof).upstreamVerified + ).toBeUndefined(); + } + expect( + parseNativeHttpsHeaders( + `HTTP/1.1 404 Not Found\r\nX-Hack-Upstream-Proof: ${proof}\r\n\r\n`, + proof + ) + ).toEqual({ statusCode: 404, upstreamVerified: true }); + expect(() => + parseNativeHttpsHeaders( + `HTTP/1.1 404 Not Found\r\nX-Hack-Upstream-Proof: ${proof}\r\nx-hack-upstream-proof: ${proof}\r\n\r\n`, + proof + ) + ).toThrow(); +}); + test("HTTPS headers permit unrelated duplicates but keep Location unambiguous", () => { expect( parseNativeHttpsHeaders( diff --git a/tests/native-project-routing.test.ts b/tests/native-project-routing.test.ts index 6e26452e9..3bc87ab9a 100644 --- a/tests/native-project-routing.test.ts +++ b/tests/native-project-routing.test.ts @@ -162,3 +162,64 @@ test("active reviewed routes request pool allocation with matching existing prob }).flags ).toEqual([]); }); + +test("a route without an authored healthcheck accepts only its exact reviewed inferred probe", () => { + const inferred = { + policy: "route", + port: 3000, + path: "/", + interval_ms: 1000, + timeout_ms: 1000, + retries: 3, + start_period_ms: 300_000, + }; + const opts = { + plan: { + services: { + web: { + ...service, + healthcheck: { disabled: false, native_http: inferred }, + }, + }, + }, + specs: { web: { labels } }, + capacity: 1, + }; + expect(reviewedNativeRoutes(opts).flags).toEqual([ + "--route-slot", + "web=auto", + ]); + for (const authoredHealthcheck of [ + null, + {}, + { disable: true }, + { test: ["CMD", "true"] }, + healthcheck, + ]) { + expect(() => + reviewedNativeRoutes({ + ...opts, + specs: { web: { labels, healthcheck: authoredHealthcheck } }, + }) + ).toThrow(); + } + for (const changed of [ + { ...inferred, port: 3001 }, + { ...inferred, path: "/health" }, + { ...inferred, policy: "anything" }, + { ...inferred, timeout_ms: 60_000 }, + { ...inferred, extra: true }, + { port: 3000 }, + ]) { + expect(() => + reviewedNativeRoutes({ + ...opts, + plan: { + services: { + web: { ...service, healthcheck: { native_http: changed } }, + }, + }, + }) + ).toThrow(); + } +}); diff --git a/tests/native-project-start.test.ts b/tests/native-project-start.test.ts index c2e0d539e..9dd993a53 100644 --- a/tests/native-project-start.test.ts +++ b/tests/native-project-start.test.ts @@ -3157,6 +3157,82 @@ test("HTTPS startup refuses a non-2xx reviewed health response", async () => { expect(events).toContain("https-close"); }); +test("inferred HTTPS readiness accepts 2xx-4xx without following redirects and rejects 5xx", async () => { + const probe = { + policy: "route", + port: 3000, + path: "/", + interval_ms: 1000, + timeout_ms: 1000, + retries: 3, + start_period_ms: 300_000, + }; + const plan = { + services: { + web: { + routing: { port: 3000, hostnames: ["web.example.hack"] }, + healthcheck: { disabled: false, native_http: probe }, + }, + }, + }; + for (const statusCode of [200, 204, 299, 301, 399, 400, 404, 499]) { + const calls: string[] = []; + await verifyHttpsRoutes( + { + httpsPort: 18_443, + verifyHostname: async (hostname, path) => { + calls.push(hostname); + expect(path).toBe("/"); + return { + statusCode, + location: "https://foreign.invalid/", + upstreamVerified: true, + }; + }, + }, + plan, + new Set(["web"]) + ); + expect(calls).toEqual(["web.example.hack"]); + } + for (const statusCode of [199, 500, 503, 599, 600]) { + await expect( + verifyHttpsRoutes( + { + httpsPort: 18_443, + verifyHostname: async () => ({ statusCode, upstreamVerified: true }), + }, + plan, + new Set(["web"]) + ) + ).rejects.toThrow(); + } + for (const statusCode of [200, 403, 404, 421, 502]) { + await expect( + verifyHttpsRoutes( + { httpsPort: 18_443, verifyHostname: async () => ({ statusCode }) }, + plan, + new Set(["web"]) + ) + ).rejects.toThrow("UPSTREAM_RESPONSE_UNCONFIRMED"); + } + const explicit = { + services: { + web: { + ...plan.services.web, + healthcheck: { native_http: { ...probe, policy: undefined } }, + }, + }, + }; + await expect( + verifyHttpsRoutes( + { httpsPort: 18_443, verifyHostname: async () => ({ statusCode: 404 }) }, + explicit, + new Set(["web"]) + ) + ).rejects.toThrow(); +}); + test("HTTPS redirects terminate only at verified same-service reviewed aliases", async () => { const plan = { services: { From 7886d08589968ae78c0597034fe41793d4159d89 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:02:10 +0000 Subject: [PATCH 5/9] fix(native): bound inferred route readiness and probe cadence --- docs/guides/native-candidate.md | 22 ++++- docs/reference/v5-model-coverage.md | 2 +- packages/runtime-core/guest/http-probe.c | 8 +- .../runtime-core/src/project/compose/graph.rs | 9 +- .../src/provider/graph/bridges.rs | 15 ++- .../runtime-core/src/provider/http_probe.rs | 96 ++++++++++++++++++- .../runtime-core/tests/project_contract.rs | 30 ++++++ src/backends/native-http-probe-admission.ts | 4 +- src/backends/native-preflight.ts | 9 +- src/backends/native-project-routing.ts | 73 +++++++++++++- tests/native-preflight-cli.test.ts | 72 +++++++++++++- tests/native-project-routing.test.ts | 6 +- tests/native-project-start.test.ts | 4 +- 13 files changed, 314 insertions(+), 36 deletions(-) diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index cf5472da4..73deae714 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -2861,8 +2861,14 @@ shared-source mode; active project bind mounts still use the explicit project sh #### Native HTTP readiness probe For a complete Caddy route with no authored `healthcheck`, native reviews a default -probe: `GET /` on the route port, a 1000 ms interval and timeout, three consecutive -failures, and a 300000 ms startup grace matching the foreground startup deadline. +probe: `GET /` on the route port, a 1000 ms timeout, three consecutive +failures, and a 60000 ms initial startup grace. It checks every second until its +first success, then every 30 seconds for the rest of that probe generation. +After the grace, three failed startup checks make the service unhealthy before +the usual 300-second graph deadline, so startup names the failed service. +For a missing listener or HTTP 5xx during boot, check `hack logs`, fix the route +port, or add an explicit `healthcheck.x-hack-http` with your app's startup budget +and readiness path. A successful TCP connection and complete HTTP/1.0 or HTTP/1.1 response headers are required. HTTP 200–499, including `/` returning 404, is ready; 5xx, EOF, non-HTTP bytes, a timeout or partial headers are not. Redirects are not followed. At most @@ -2870,9 +2876,15 @@ four interim 1xx responses may precede the final response, within the same 8 KiB header and timeout budgets. The request uses `Host: 127.0.0.1:`. This proves endpoint readiness, not application correctness. After the first -success, three consecutive failures make the service unhealthy using the existing -probe lifecycle; source-reload downtime is not silently called healthy. Final -HTTPS publication verification accepts 2xx–4xx only with a fresh challenge added +success, three consecutive failures at the 30-second interval make the service +unhealthy. Status no longer exposes a ready route endpoint, and a new publication +or rebind cannot proceed until healthy evidence returns. This does not stop or restart the +container, withdraw an existing publication, or close its existing relay: route +traffic continues to the application and may receive its error responses. A +later successful check restores health; it does not resume one-second probing. +An inferred probe does not satisfy `depends_on: condition: service_healthy`: +that condition still requires an explicit enabled healthcheck, as on Compose. +Final HTTPS publication verification accepts 2xx–4xx only with a fresh challenge added by the upstream proxy response handler. Authority denials and routing errors do not qualify as the application's 404. No readiness decision follows an external redirect. Explicit probes retain the stricter policy below. The generated diff --git a/docs/reference/v5-model-coverage.md b/docs/reference/v5-model-coverage.md index 1718e8138..0a3ef2029 100644 --- a/docs/reference/v5-model-coverage.md +++ b/docs/reference/v5-model-coverage.md @@ -49,7 +49,7 @@ must be attached to the eventual exact integration revision. | Settled retained Compose route-attempt retirement; pinned references; interrupted deletion | **Coverage gap:** filesystem journal compaction is not modeled; native-route-stop covers a different stop protocol | Instance mutation lock and exact surviving claim witnesses in [journal compaction](../../src/lib/native-compose-route-claims.ts); [claim regressions](../../tests/native-compose-route-claims.test.ts) and [retained lifecycle integration](../../tests/native-compose-adoption-routing-generation.test.ts) | 24 successful operations keep two attempt directories and constant Linux descriptors; four integrated lifecycle operations check directory counts. Injected unlink failures cover same-store and fresh-store cleanup retries. Assumes cooperative same-user writers, the caller's mutation lock and working filesystem synchronization. No process-kill/power-loss durability proof or bound on unresolved/over-budget histories. | | Graph cleanup container stop batch: bounded in-flight stops, shared worker queue, connect-stage retry, batch-wide unreachable state, deadline expiry, worker panic | **Coverage gap:** no maintained model covers scheduling inside the stop batch; native-route-stop covers a different stop protocol | `dispatch` and `attempt` in [the stop batch](../../packages/runtime-core/src/provider/engine/stop.rs): at most 4 stops in flight, every worker joined; only `connect`/`connect_timeout` failures retry (25 ms doubling to 500 ms, within the batch deadline and 30 s of a stop's first connect failure); once one stop exhausts its window, later connect failures return at once; a deadline reached during backoff keeps the connect stage; `response`, `timeout` and `transport` never retry. Regressions in the same file's `provider::engine::stop` tests (bounded in-flight socket server, late socket, missing socket, 14-stop retry counts, persistent connect failures, oversleeping backoff, non-retried stages, worker panic) and `fourteen_service_connect_failures_keep_resumable_progress_and_retry_only_unstopped` | Tests cover exact attempt counts and the in-flight bound for fixed interleavings; no model explores all worker schedules or proves at most one delivered request per stop. Assumes a failed connect sent no request bytes and that the stop budget covers fully serialized grace periods. Graph-level resume after a partial batch is covered by tests only. Host qualification of the descriptor-pressure root cause (HACK-1302) remains open. | | Concurrent stop/publication; route claim and relay lifetime | native-route-stop, shared-https-lifetime, previous-boot-shared-https, relay-lifecycle-barrier, relay-authorization, relay-staging, relay-fence | Fenced route removal, relay owner, ready/removed protocol, authenticated observations | Native DNS/TLS/browser behavior; old-boot uncertainty and external routing. | -| Inferred Caddy route readiness without authored probes | Existing probe generation, exec identity and freshness checks are reused; no new lifecycle transition | `http_probe.rs` and compiled `guest/http-probe.c` tests cover TCP refusal, complete HTTP 2xx–4xx, 5xx, EOF, timeout, partial/oversized headers and bounded interim responses. Project contracts preserve explicit checks; TypeScript route/preflight tests require the exact reviewed inferred policy. HTTPS tests require a fresh upstream response challenge for inferred routes | 32 route slots, 1 s interval/deadline, 8 KiB headers, four interim responses, three failed samples and 300 s initial grace. No model explores HTTP parsing or Caddy handler behavior. TLS fixtures simulate the proxy marker; an actual Caddy/authority/VM run must qualify application 404 versus authority denial, SNI mismatch and missing upstream. | +| Inferred Caddy route readiness without authored probes | Existing probe generation, exec identity and freshness checks are reused; no new lifecycle transition | `http_probe.rs` and compiled `guest/http-probe.c` tests cover TCP refusal, complete HTTP 2xx–4xx, 5xx, EOF, timeout, partial/oversized headers and bounded interim responses. Compiled-helper startup regressions cover no listener and boot-time 5xx reaching a named service failure before the graph deadline; a cadence regression covers the switch after first success. Bridge reservation regressions preserve running relay assignments when ready endpoints disappear. Project contracts preserve explicit checks and refuse inferred service_healthy dependencies; TypeScript route/preflight tests require the exact reviewed inferred policy. HTTPS tests require a fresh upstream response challenge for inferred routes | 32 route slots, 1 s startup/30 s steady interval, 1 s deadline, 8 KiB headers, four interim responses, three failed samples and 60 s initial grace. Bridge registry tests do not exercise continued route traffic; that remains a real-Caddy/VM check. No model explores HTTP parsing or Caddy handler behavior. TLS fixtures simulate the proxy marker; an actual Caddy/authority/VM run must qualify application 404 versus authority denial, SNI mismatch and missing upstream. | | Shared host helper instance: overlapping ups, session adoption, stale create decisions, replacement before finalize, epoch-record failure, interrupted cleanup, concurrent down, displaced launches, failed down session stop | shared-helper-instance | `src/lib/project-shared-helpers.ts` (shutdown generation, `assertCurrent`, `SharedHelperSessionOwnership`); `src/lib/project-lifecycle-controller.ts` session decision, the locked creation transaction (revalidate, publish, create, environment, record, stop on failure, all under one lock), launches fenced by the lock and a token check, abort and signal cleanup through the same epoch decision (`runIfOwned`), an unknown exact-presence answer failing the inspection, and the locked conditional finalize; `stopSessionBeforeReleasingHelpers` (down, restart, native retirement keep holders when the session stop fails); `runMuxCommand` bounds every lifecycle and backend mux call so it settles before the lock is released, and reports a timeout as unknown (never absence); `tests/mux-command-timeout.test.ts`; `tests/project-lifecycle-controller-ownership.test.ts`, `tests/project-shared-helpers.test.ts`, `tests/native-project-down.test.ts`, e2e `shared-host-helpers` | Helper holder/rollback and claim protocol are not modeled (claim fixtures cover them); timeout settlement is an assumption the code does not guarantee (a killed tmux client's queued command may still apply, so a late kill or window command can reach a newer session; consequences and the session-ID fence, not in this slice, are in the README); zellij is refused for shared-helper lifecycle sessions (token written after creation); the gap between a down's session stop and its release (an up that captured after the stopping mark can still succeed there) is a documented first-slice limitation; real mux/process/lock-file behavior. | | Engine restart policies: stop versus pending restart, guest engine restart, stale incarnations, terminal proof | service-restart-policy | Receipt-admitted policy, disarm-before-stop and `restarting` stop preflight in `shutdown.rs`; admission refusals for incarnation-bound delivery classes in `restart_policy.rs`; owner-driven `Rebind` after full retirement, fresh ownership/attachment checks and a new reservation in `bridges/restart_rebind.rs`; route probe sample-generation rearm; Rust tests for admission, endpoint identity, port-busy refusal, disarm and stop preflight; portable selected-child readiness guard plus macOS two-real-child polling regression prevent historical sibling exits from rejecting a replacement or misclassifying its own listener | Real guest engine behavior (crash restart, stop during backoff, disarm before deletion) needs native host evidence; several services, selected-child readiness during sibling failure, backoff timing and retry counting are not modeled. Same-port/route reachability after a policy restart, route probe rearm and per-slot child retirement still require real-VM host evidence. Rebind abstracts retirement/reservation/publication; relay lifecycle, fence and staging models cover their separate failure boundaries. | | Foreground Ctrl-C down hooks: concurrent `down`, second interrupt, crash between or during hooks, stop during `up.after`, next generation | native-foreground-down-hooks | Value-free compare-and-swap journal in [down hooks](../../src/backends/native-project-down-hooks.ts), foreground orchestration in [foreground stop](../../src/backends/native-project-foreground-stop.ts) and `hack down` in [native down](../../src/backends/native-project-down.ts); [journal](../../tests/native-project-down-hooks.test.ts), [foreground](../../tests/native-project-start-stop-hooks.test.ts) and [down](../../tests/native-project-down-journal.test.ts) regressions | Durable writes are atomic in the model; real fsync/power loss, a stale journal lock after a crash inside the lock, and the live process-group behavior of the graph owner on a host are not covered. One generation and at most two `down` invocations are explored. | diff --git a/packages/runtime-core/guest/http-probe.c b/packages/runtime-core/guest/http-probe.c index 25f2c4e8f..5d27f754b 100644 --- a/packages/runtime-core/guest/http-probe.c +++ b/packages/runtime-core/guest/http-probe.c @@ -174,10 +174,14 @@ int main(int argc, char **argv) { // On the first request only, health == 1 means success even during startup grace. if (sequence == 1 && !publish(directory, "first-sample", argv[8], sequence, health, failures, finished - begun)) return 125; if (!publish(directory, "status", argv[8], sequence, health, failures, finished - begun)) return 125; - uint64_t next = begun + interval; + // Generated route probes check quickly during initial compilation, then + // use the reviewed steady interval for the rest of this exec generation. + // An unhealthy sample never re-enables the fast startup cadence. + uint64_t period = route && !ever_healthy && interval > 1000 ? 1000 : interval; + uint64_t next = begun + period; while (!stopping && millis(CLOCK_MONOTONIC) < next) { uint64_t delay = next - millis(CLOCK_MONOTONIC); - if (delay > interval) break; + if (delay > period) break; struct timespec pause = {.tv_sec = (time_t)(delay / 1000), .tv_nsec = (long)(delay % 1000) * 1000000}; if (nanosleep(&pause, NULL) && errno != EINTR) return 125; } diff --git a/packages/runtime-core/src/project/compose/graph.rs b/packages/runtime-core/src/project/compose/graph.rs index 250ae11ab..759aeb3c9 100644 --- a/packages/runtime-core/src/project/compose/graph.rs +++ b/packages/runtime-core/src/project/compose/graph.rs @@ -69,10 +69,11 @@ pub(super) fn validate( )); } if condition.condition == "service_healthy" - && !target - .healthcheck - .as_ref() - .is_some_and(|h| !h.disabled && (h.test.is_some() || h.native_http.is_some())) + && !target.healthcheck.as_ref().is_some_and(|h| { + !h.disabled + && h.native_http.as_ref().is_none_or(|p| p.policy.is_none()) + && (h.test.is_some() || h.native_http.is_some()) + }) { diagnostics.push(Diagnostic::error("missing_healthcheck",&field,"A healthy dependency needs an explicit enabled healthcheck; image health defaults are not assumed.")); } diff --git a/packages/runtime-core/src/provider/graph/bridges.rs b/packages/runtime-core/src/provider/graph/bridges.rs index d8ce59ad4..d203c4352 100644 --- a/packages/runtime-core/src/provider/graph/bridges.rs +++ b/packages/runtime-core/src/provider/graph/bridges.rs @@ -1224,7 +1224,7 @@ mod tests { assert_eq!(persisted.slots[&0].run, "b".repeat(32)); } #[test] - fn batch_commit_is_atomic_across_late_endpoint_failure_and_success() { + fn missing_ready_endpoints_refuse_new_routes_without_retiring_running_relays() { let fixture_root = super::super::tests::Fixture::new(); let candidate = Candidate::discover(&fixture_root.0).unwrap(); let owner = "a".repeat(32); @@ -1233,6 +1233,17 @@ mod tests { initialize_registry(&candidate, &owner, 3).unwrap(); let mut store = fixture(); store.owner = owner.clone(); + let existing = store.slots.get_mut(&0).unwrap(); + existing.phase = "running".into(); + existing.relay = Some(relay::Relay { + transport: relay::Transport::ReservationV1, + launch_serial: 1, + binary_sha256: "f".repeat(64), + target_pid: 42, + target_start: 123, + port: 3000, + }); + store.next_launch_serial = 1; save(&candidate, &store).unwrap(); let before = fs::read(root(&candidate).join("state.json")).unwrap(); let requested = BTreeMap::from([ @@ -1303,6 +1314,8 @@ mod tests { assert_eq!(reserved["web"].slot, 2); let persisted = read_owner_registry(&candidate, &owner, 3, false).unwrap(); assert_eq!(persisted.slots.len(), 3); + assert_eq!(persisted.slots[&0].phase, "running"); + assert_eq!(persisted.slots[&0].relay.as_ref().unwrap().launch_serial, 1); for (service, route) in reserved { assert_eq!(persisted.slots[&route.slot].run, run); assert_eq!(persisted.slots[&route.slot].service, service); diff --git a/packages/runtime-core/src/provider/http_probe.rs b/packages/runtime-core/src/provider/http_probe.rs index 76534a784..725d048e9 100644 --- a/packages/runtime-core/src/provider/http_probe.rs +++ b/packages/runtime-core/src/provider/http_probe.rs @@ -42,10 +42,10 @@ impl HttpProbe { Self { port, path: "/".into(), - interval_ms: 1_000, + interval_ms: 30_000, timeout_ms: 1_000, retries: 3, - start_period_ms: 300_000, + start_period_ms: 60_000, policy: Some(HttpProbePolicy::Route), } } @@ -250,6 +250,14 @@ mod tests { ); let inferred = HttpProbe::inferred_route(3000); inferred.validate().unwrap(); + // Even a timeout on every startup sample exhausts grace and retries + // before the frontend's 300-second graph budget. Startup cadence is at + // most one second; the steady interval is thirty seconds. + assert_eq!(inferred.interval_ms, 30_000); + assert!( + inferred.start_period_ms + u64::from(inferred.retries) * (1_000 + inferred.timeout_ms) + < 300_000 + ); assert_eq!(serde_json::to_value(&inferred).unwrap()["policy"], "route"); assert_eq!( inferred @@ -304,6 +312,15 @@ mod tests { grace: u64, timeout: u64, policy: Option, + ) -> Self { + Self::with_cadence(mode, grace, timeout, policy, 100) + } + fn with_cadence( + mode: usize, + grace: u64, + timeout: u64, + policy: Option, + interval: u64, ) -> Self { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let port = listener.local_addr().unwrap().port(); @@ -377,6 +394,7 @@ mod tests { config.port = port; config.start_period_ms = grace; config.timeout_ms = timeout; + config.interval_ms = interval; config.policy = policy; if policy == Some(HttpProbePolicy::Route) { config.path = "/".into(); @@ -419,7 +437,10 @@ mod tests { Some((health, raw.split(' ').nth(2).unwrap().parse().unwrap())) } fn wait(&mut self, health: Health, after: u64) -> u64 { - let deadline = Instant::now() + Duration::from_secs(4); + self.wait_for(health, after, Duration::from_secs(4)) + } + fn wait_for(&mut self, health: Health, after: u64, budget: Duration) -> u64 { + let deadline = Instant::now() + budget; loop { assert!( self.process.as_mut().unwrap().try_wait().unwrap().is_none(), @@ -562,6 +583,75 @@ mod tests { b"retained publication" ); } + #[test] + fn inferred_boot_failures_name_the_service_before_the_graph_deadline() { + use crate::project::execution::{ + self, Condition, Driver, Event, Graph, Observation, Service, + }; + use std::collections::BTreeMap; + struct Probed(Fixture); + impl Driver for Probed { + fn record(&mut self, _: Event<'_>) -> Result<(), CandidateError> { + Ok(()) + } + fn start(&mut self, _: &str) -> Result<(), CandidateError> { + Ok(()) + } + fn observe(&mut self, _: &str) -> Result { + Ok(Observation::Running { + health: self.0.sample().map_or(Health::Starting, |s| s.0), + }) + } + } + for listening in [false, true] { + // Exercise the compiled production helper, with the same grace + // and retry rules scaled down; one case has no TCP listener, + // the other keeps answering HTTP 500 through startup. + let mut fixture = Fixture::with_policy(9, 200, 40, Some(HttpProbePolicy::Route)); + if !listening { + fixture.stop.store(1, Ordering::SeqCst); + fixture.server.take().unwrap().join().unwrap(); + } + let mut driver = Probed(fixture); + let graph = Graph::from_services(BTreeMap::from([( + "web".into(), + Service { + dependencies: BTreeMap::new(), + ready: Condition::Healthy, + }, + )])) + .unwrap(); + let failure = + execution::run(&graph, &mut driver, Duration::from_secs(3)).unwrap_err(); + assert_eq!(failure.code, "graph_service_failed"); + assert_eq!(failure.service.as_deref(), Some("web")); + assert!(failure.message.contains("web") && failure.message.contains("unhealthy")); + if listening { + assert!(driver.0.accepted.load(Ordering::SeqCst) >= 3); + } + } + } + + #[test] + fn inferred_probe_slows_after_first_success_and_does_not_resume_fast_polling() { + // Shorten only the steady interval to keep this real-helper test + // bounded. Production admission pins it to 30 seconds. + let mut f = Fixture::with_cadence(9, 20_000, 40, Some(HttpProbePolicy::Route), 6_000); + let starting = f.wait(Health::Starting, 1); + f.mode.store(7, Ordering::SeqCst); + let ready = f.wait(Health::Healthy, starting); + f.mode.store(9, Ordering::SeqCst); + thread::sleep(Duration::from_millis(1_250)); + assert_eq!(f.sample().unwrap(), (Health::Healthy, ready)); + let down = f.wait_for(Health::Unhealthy, ready, Duration::from_secs(22)); + assert!(down >= ready + 3); + // Unhealthy status does not return to the one-second startup loop. + thread::sleep(Duration::from_millis(1_250)); + assert_eq!(f.sample().unwrap(), (Health::Unhealthy, down)); + f.mode.store(7, Ordering::SeqCst); + f.wait_for(Health::Healthy, down, Duration::from_secs(8)); + } + #[test] fn inferred_route_accepts_http_2xx_through_4xx_but_never_tcp_only_or_http_5xx() { for (mode, expected) in [ diff --git a/packages/runtime-core/tests/project_contract.rs b/packages/runtime-core/tests/project_contract.rs index b35180501..63ebfbd98 100644 --- a/packages/runtime-core/tests/project_contract.rs +++ b/packages/runtime-core/tests/project_contract.rs @@ -93,6 +93,36 @@ fn standard_route_without_healthcheck_infers_a_reviewed_probe_without_editing_so assert!(!fixture.candidate.state_root.exists()); } +#[test] +fn inferred_route_probe_does_not_satisfy_compose_healthy_dependencies() { + let route = format!( + "{BASIC} labels:\n caddy: web.example.hack\n caddy.reverse_proxy: '{{{{upstreams 3000}}}}'\n caddy.tls: internal\n" + ); + for condition in ["service_healthy", "service_started"] { + let dependency = format!( + " app:\n image: alpine:3.21\n depends_on:\n web: {{condition: {condition}}}\n" + ); + let fixture = Fixture::new(&format!("{route}{dependency}")); + let report = fixture.plan(); + let missing = report + .plan + .diagnostics + .iter() + .any(|d| d.code == "missing_healthcheck"); + assert_eq!(missing, condition == "service_healthy"); + assert_eq!( + report.plan.enrollment_compatible, + condition != "service_healthy" + ); + if condition == "service_healthy" { + let explicit = " healthcheck:\n x-hack-http: {port: 3000, path: /health, interval_ms: 30000, timeout_ms: 1000, retries: 3, start_period_ms: 0}\n"; + let fixture = Fixture::new(&format!("{route}{explicit}{dependency}")); + assert!(fixture.plan().plan.enrollment_compatible); + assert!(!fixture.codes().contains(&"missing_healthcheck".into())); + } + } +} + #[test] fn route_inference_never_replaces_authored_healthchecks_or_accepts_an_authored_policy() { let route = format!( diff --git a/src/backends/native-http-probe-admission.ts b/src/backends/native-http-probe-admission.ts index a277009df..76695d19c 100644 --- a/src/backends/native-http-probe-admission.ts +++ b/src/backends/native-http-probe-admission.ts @@ -27,10 +27,10 @@ export function isInferredNativeRouteProbe( probe.policy === "route" && probe.port === port && probe.path === "/" && - probe.interval_ms === 1000 && + probe.interval_ms === 30_000 && probe.timeout_ms === 1000 && probe.retries === 3 && - probe.start_period_ms === 300_000 + probe.start_period_ms === 60_000 ); } function bounded(value: unknown, min: number, max: number): boolean { diff --git a/src/backends/native-preflight.ts b/src/backends/native-preflight.ts index 1cbd5c8bb..04c7b5ed2 100644 --- a/src/backends/native-preflight.ts +++ b/src/backends/native-preflight.ts @@ -30,6 +30,7 @@ import { } from "./native-project-ports.ts"; import { nativeBridgeCapacity, + nativeRouteProbeRefusalDetails, reviewedNativeRoutes, } from "./native-project-routing.ts"; import { @@ -1403,7 +1404,7 @@ function routeFindings(opts: { specs, capacity: 1, }); - } catch { + } catch (error) { found.push( finding( { @@ -1413,11 +1414,7 @@ function routeFindings(opts: { service: name, field: "healthcheck.x-hack-http", }, - { - message: - "The route lacks matching reviewed readiness, has an incompatible explicit healthcheck, or also declares a dependency cache.", - fix: "Keep the default inferred route probe, or give an explicit healthcheck x-hack-http on the route port. Disabled or command-only healthchecks and dependency-cache routes still refuse.", - } + nativeRouteProbeRefusalDetails(error) ) ); } diff --git a/src/backends/native-project-routing.ts b/src/backends/native-project-routing.ts index 586095ba4..8076c594f 100644 --- a/src/backends/native-project-routing.ts +++ b/src/backends/native-project-routing.ts @@ -14,6 +14,50 @@ function refused(): Error { "Native routes require reviewed Caddy labels and matching native route readiness; explicit healthchecks are never replaced.", }); } + +const ROUTE_PROBE_REFUSALS = { + disabled: { + message: "The routed service explicitly disables its healthcheck.", + fix: "Replace the disabled healthcheck with an x-hack-http-only healthcheck on the route port, or remove the healthcheck declaration to use the inferred probe.", + }, + command: { + message: + "The routed service has a command-only healthcheck; native route readiness requires an HTTP probe.", + fix: "Replace the command healthcheck with an x-hack-http-only healthcheck on the route port, or remove the healthcheck declaration to use the inferred probe.", + }, + cache: { + message: + "The routed service also declares a dependency cache, which native routing does not support.", + fix: "Move the dependency cache to a separate non-routed service, or remove the dependency cache declaration.", + }, + mismatch: { + message: + "The route has no valid reviewed HTTP probe matching its route port and authored healthcheck.", + fix: "Check the route port and healthcheck.x-hack-http; remove the authored healthcheck to infer readiness, or configure a valid explicit HTTP probe on that port. If the review still differs, update Hack and rerun native preflight.", + }, +} as const; + +/** Closed reasons shared by startup admission and preflight; no authored values. */ +class RouteProbeRefusal extends HackCliError { + readonly reason: keyof typeof ROUTE_PROBE_REFUSALS; + constructor(reason: keyof typeof ROUTE_PROBE_REFUSALS) { + const detail = ROUTE_PROBE_REFUSALS[reason]; + super({ + code: "E_NATIVE_PROJECT_UNSUPPORTED", + message: `${detail.message} ${detail.fix}`, + }); + this.reason = reason; + } +} + +export function nativeRouteProbeRefusalDetails(error: unknown): { + readonly message: string; + readonly fix: string; +} { + return ROUTE_PROBE_REFUSALS[ + error instanceof RouteProbeRefusal ? error.reason : "mismatch" + ]; +} /** * Hack's authored one-shot marker. Only this exact key with the literal value `true` * (map form) or the exact entry `hack.service.one-shot=true` (list form) is admitted, @@ -142,6 +186,28 @@ function requireRouteProbe( service: Readonly>, spec: Readonly> | undefined ): void { + if ( + service.dependency_cache !== undefined && + service.dependency_cache !== null + ) { + throw new RouteProbeRefusal("cache"); + } + const authored = spec?.healthcheck; + if ( + (isRecord(service.healthcheck) && service.healthcheck.disabled === true) || + (isRecord(authored) && + (authored.disable === true || + (Array.isArray(authored.test) && authored.test[0] === "NONE"))) + ) { + throw new RouteProbeRefusal("disabled"); + } + if ( + isRecord(authored) && + !Object.hasOwn(authored, "x-hack-http") && + Object.hasOwn(authored, "test") + ) { + throw new RouteProbeRefusal("command"); + } const probe = spec && isRecord(spec.healthcheck) ? spec.healthcheck["x-hack-http"] @@ -168,12 +234,9 @@ function requireRouteProbe( probe.port === service.routing.port && reviewedProbe.policy === undefined) ) || - reviewedProbe.port !== service.routing.port || - (isRecord(service.healthcheck) && service.healthcheck.disabled === true) || - (service.dependency_cache !== undefined && - service.dependency_cache !== null) + reviewedProbe.port !== service.routing.port ) { - throw refused(); + throw new RouteProbeRefusal("mismatch"); } } export function reviewedNativeRoutes(opts: { diff --git a/tests/native-preflight-cli.test.ts b/tests/native-preflight-cli.test.ts index 5d35d9684..8833abd35 100644 --- a/tests/native-preflight-cli.test.ts +++ b/tests/native-preflight-cli.test.ts @@ -713,6 +713,74 @@ test( SPAWN_TIMEOUT_MS * 2 ); +test( + "route preflight gives a separate cause and remedy for each refused probe", + async () => { + const probe = { + port: 3000, + path: "/health", + interval_ms: 1000, + timeout_ms: 1000, + retries: 3, + start_period_ms: 0, + }; + for (const entry of [ + { + authored: { disable: true }, + reviewed: { healthcheck: { disabled: true } }, + message: "explicitly disables", + fix: "Replace the disabled healthcheck", + }, + { + authored: { test: ["CMD", "true"] }, + reviewed: { healthcheck: { test: ["CMD", "true"] } }, + message: "command-only healthcheck", + fix: "Replace the command healthcheck", + }, + { + authored: { "x-hack-http": probe }, + reviewed: { healthcheck: { native_http: probe }, dependency_cache: {} }, + message: "also declares a dependency cache", + fix: "Move the dependency cache", + }, + { + authored: { "x-hack-http": probe }, + reviewed: { healthcheck: { native_http: { ...probe, port: 3001 } } }, + message: "no valid reviewed HTTP probe matching", + fix: "Check the route port", + }, + ]) { + const project = await fixture({ + services: { + web: { + image: IMAGE, + labels: { + caddy: "web.example.test", + "caddy.reverse_proxy": "{{upstreams 3000}}", + "caddy.tls": "internal", + }, + healthcheck: entry.authored, + }, + }, + }); + await project.review([], { + services: { + web: { active: true, routing: { port: 3000 }, ...entry.reviewed }, + }, + }); + const result = await preflight(project, { ...project.native, ...GRANT }); + expect(result.code).toBe(1); + const found = result.report.blockers.find( + (f) => f.code === "route_probe_required" + ); + expect(found?.service).toBe("web"); + expect(found?.message).toContain(entry.message); + expect(found?.fix).toContain(entry.fix); + } + }, + SPAWN_TIMEOUT_MS * 2 +); + test( "route admission, project-wide hosts and per-environment dependency files are checked as native up checks them", async () => { @@ -754,10 +822,10 @@ test( policy: "route", port: 3000, path: "/", - interval_ms: 1000, + interval_ms: 30_000, timeout_ms: 1000, retries: 3, - start_period_ms: 300_000, + start_period_ms: 60_000, }, }, }, diff --git a/tests/native-project-routing.test.ts b/tests/native-project-routing.test.ts index 3bc87ab9a..0435d2510 100644 --- a/tests/native-project-routing.test.ts +++ b/tests/native-project-routing.test.ts @@ -142,7 +142,7 @@ test("active reviewed routes request pool allocation with matching existing prob } expect(refusal).toMatchObject({ code: "E_NATIVE_PROJECT_UNSUPPORTED", - message: expect.stringContaining("reviewed Caddy labels"), + message: expect.stringContaining("no valid reviewed HTTP probe matching"), }); expect(() => reviewedNativeRoutes({ @@ -168,10 +168,10 @@ test("a route without an authored healthcheck accepts only its exact reviewed in policy: "route", port: 3000, path: "/", - interval_ms: 1000, + interval_ms: 30_000, timeout_ms: 1000, retries: 3, - start_period_ms: 300_000, + start_period_ms: 60_000, }; const opts = { plan: { diff --git a/tests/native-project-start.test.ts b/tests/native-project-start.test.ts index 9dd993a53..588d8fa35 100644 --- a/tests/native-project-start.test.ts +++ b/tests/native-project-start.test.ts @@ -3162,10 +3162,10 @@ test("inferred HTTPS readiness accepts 2xx-4xx without following redirects and r policy: "route", port: 3000, path: "/", - interval_ms: 1000, + interval_ms: 30_000, timeout_ms: 1000, retries: 3, - start_period_ms: 300_000, + start_period_ms: 60_000, }; const plan = { services: { From 24726f3e144b1093d8d9daefb674f8dead1098f6 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:46:19 +0000 Subject: [PATCH 6/9] fix(native): wait for bridge sockets after provider readiness --- docs/diagrams/native-bridge-startup.mmd | 23 ++ docs/diagrams/native-bridge-startup.svg | 66 +++++ docs/guides/native-candidate.md | 11 + docs/reference/v5-model-coverage.md | 1 + packages/runtime-core/README.md | 22 ++ .../runtime-core/src/provider/artifact.rs | 3 + packages/runtime-core/src/provider/bridge.rs | 110 +++++++- .../src/provider/bridge_startup_tests.rs | 244 ++++++++++++++++++ .../runtime-core/src/provider/lifecycle.rs | 8 + tests/native-project-start.test.ts | 34 +++ tests/native-runtime-client.test.ts | 14 + 11 files changed, 524 insertions(+), 12 deletions(-) create mode 100644 docs/diagrams/native-bridge-startup.mmd create mode 100644 docs/diagrams/native-bridge-startup.svg create mode 100644 packages/runtime-core/src/provider/bridge_startup_tests.rs diff --git a/docs/diagrams/native-bridge-startup.mmd b/docs/diagrams/native-bridge-startup.mmd new file mode 100644 index 000000000..7cde21361 --- /dev/null +++ b/docs/diagrams/native-bridge-startup.mmd @@ -0,0 +1,23 @@ +--- +title: Native bridge startup +--- +flowchart TD + ready[Guest ready] --> check[Verify provider] + check --> inspect[Inspect all sockets] + check -->|identity changed| fail[Refuse startup] + inspect -->|unsafe identity| fail + inspect -->|ready or pending| recheck[Recheck provider] + recheck -->|identity changed| fail + recheck --> deadline{Before 10 seconds?} + deadline -->|no| fail + deadline -->|yes| complete{All sockets private?} + complete -->|no| wait[Wait up to 50ms] + wait --> check + complete -->|yes| audit[Full audit then adoption] + fail --> stop[Verified provider stop] + stop --> retained[Retain failure evidence] + class ready,check,inspect,recheck live + class wait wait + class audit ok + class fail fail + class retained idle diff --git a/docs/diagrams/native-bridge-startup.svg b/docs/diagrams/native-bridge-startup.svg new file mode 100644 index 000000000..9977810fd --- /dev/null +++ b/docs/diagrams/native-bridge-startup.svg @@ -0,0 +1,66 @@ +Native bridge startupidentity changedunsafe identityready or pendingidentity changednoyesnoyesGuest readyVerify providerInspect all socketsRefuse startupRecheck providerBefore 10 seconds?All sockets private?Wait up to 50msFull audit then adoptionVerified provider stopRetain failure evidence \ No newline at end of file diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index 67d63f56b..009e3a623 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -699,6 +699,17 @@ within the macOS length limit. A clean `runtime down` keeps this alias, so no ot local user can take its name while the pool is stopped. It removes only the provider socket files the stop proved closed, through the real data directory. +On a fresh pool boot or a boot after stopping the pool, native waits up to 10 seconds +for every declared bridge socket to appear with the required private ownership and mode. +The provider's guest-ready signal can arrive before its bridge threads bind. Native +rechecks the same provider identity around each observation; it never accepts an unsafe +socket, changes its permissions or removes it to make startup pass. Existing running-pool +audits do not wait. A socket timeout or identity refusal takes the verified failed-boot +stop path and names the recovery command: run the selected native executor's +`--candidate-root "$HACK_NATIVE_HOME" runtime down --recover --json`, then retry. This preserves +retained disks and records an unclean stop; it does not claim the guest shut down cleanly. +`hack down` may say the project was not started because no project mapping was published. + Host cleanup or a reboot can still remove the alias. For a clean stop, observation commands such as `runtime status` accept the absent alias and create nothing, and the next `runtime up`, `runtime recover` or approved-host update restores the exact alias diff --git a/docs/reference/v5-model-coverage.md b/docs/reference/v5-model-coverage.md index 1dd7bf9d5..5a4db2391 100644 --- a/docs/reference/v5-model-coverage.md +++ b/docs/reference/v5-model-coverage.md @@ -36,6 +36,7 @@ must be attached to the eventual exact integration revision. | Property and critical states | Model coverage | Implementation/regression boundary | Remaining qualification | | --- | --- | --- | --- | +| Guest control ready before exposed bridge sockets bind; identity changes or the startup deadline expires | No new TLA+ model; deterministic clock schedules both readiness orders against real Unix socket nodes | `provider/bridge.rs` waits only in fresh `finish_boot`, before adoption, and keeps running audits immediate. `bridge_startup_tests.rs` covers two slots, late creation and mode tightening, permanent absence/unsafe modes, type/UID/link refusals, provider loss before/after observation and late success rejection. No wait mutates sockets or grants cleanup authority; the existing verified-provider stop handles failure. `native-runtime-client.test.ts` checks the fixed recovery command reaches the frontend | Two sockets, finite scheduled events, injected provider verification and monotonic clock. Kernel process identity, provider scheduling, disk durability and successful recovery remain host boundaries. Require ten consecutive real-VM Mac Ctrl-C scenarios with pool stop/start; portable tests are not that evidence. | | CI cancellation must not lose merged-change coverage; only exact-head full evidence may qualify a prerelease | No maintained GitHub scheduler model. Finite fixture interleavings cover predecessor success/failure/cancellation, two competing full/scoped runs and two rerun attempts | `ci.yml` shares only the next-push cancellation group and requires a successful before run for selection. `selectPrereleaseCiRun` reads current attempts; `requireBranchChecks` binds suite and job check IDs. `ci-trusted-policy`, `prerelease-plan` and `release-prepare-gate` regressions reject cancelled predecessors, green subsets, stale checks, skipped hosts and changed attempts | API snapshots and GitHub group/event semantics are assumptions; races after the last read remain possible. Publication rechecks the frozen protected head and full evidence but does not transactionally lock GitHub. Hosted dispatch/cancellation checks qualify wiring; no scheduler liveness or formal refinement claim. | | Native log follower versus lifecycle mutation, frontend loss, stopped container and transport loss | No new state model: the follower observes without a mutation lease and grants no cleanup authority | `provider/graph/log_follow.rs` and its native counterpart pin Ready admission, container and start generation; `provider/engine/log_follow.rs` bounds framing and distinguishes real EOF from stopped-service idle cutoff. `lifecycle_log_follow_tests.rs` keeps the production transport on a fake Engine socket active while acquiring startup and operation leases, with a same-lock contention control and output while each lease is held. `native-log-follow-source.test.ts` guards the observer-only route; process tests cover cancellation, frontend loss, real output-pipe closure while idle, replacement-container restart (including retained adopted run/owner IDs) with one marker, NDJSON admission failures, truncated/partial output and slow idle-end delivery. `native-log-restart.test.ts` covers bounded re-admission, unchanged-container refusal and prompt down; `native-log-selection.test.ts` covers strict finite admission responses and owned-child cancellation/timeouts; advisory markers grant no authority | 4 KiB payload envelopes, 8 KiB wire lines, 64 Ki-character retained payload prefixes excluding Engine timestamps, 30-second restart wait, 1000-line new-container backlog. The portable lease test fakes identity/state checks; it does not exercise host process/disk auditing, graph admission or the VM relay. Real relay EOF on stop/down/restart, VM loss and concurrent lifecycle latency require Apple Silicon acceptance. No performance or whole-product parity claim. | | Foreground group interrupt; publisher exit before authenticated stop; dead owner's pending cleanup enrollment and bridge release | native-publisher-stop | [Process adapter](../../src/backends/native-project-process.ts) isolates Compose owners regardless of hooks; Rust `process::publisher_command` isolates publishers; `legacy_cleanup_enrollment` binds the old coordinator and inventories before a distinct recovery intent. [Group regression](../../tests/native-foreground-group-interrupt.test.ts) signals the whole group for fresh/restore owners; Rust real-publisher process, frontend-loss, exact child-reaping and enrollment tests check grouping, lifetime wakeups and stale selectors. | One graph, one publisher, one crash, one identity replacement and one audited successor boot; 168 states. Publisher exit observation is separate from lease-bound reaping/proof commitment; `NoUncommittedExit` rejects owner cleanup while proof is pending. A busy lease stutters without delaying stop control. SIGHUP requests retaining cleanup; a reaped publisher enters an observable publisher-down state until explicit stop. Native process tests and frontend framing/ps tests cover signal delivery and notices. Dependency release is a separate post-completion step; `NoStrandedReservation` rejects success with a held reservation. Real reservation-file regressions check retry, capacity reuse and refusal fences. Separate stopped-receipt and intent-completion commits are collapsed; TypeScript dispatch and Rust completion-digest regressions cover that interruption window. Publication-registry sync/rename is abstracted and covered by portable commit-interruption tests. Kernel identity, socket/inode proofs, guest observations, durable storage and existing next.13 home recovery remain implementation/Apple Silicon VM gates. No eventual recovery is asserted. | diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index 69385d9e9..e23749307 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -581,6 +581,28 @@ digest is. A mismatch refuses with `disk_template_untrusted` and changes no file removing the refused file lets SmolVM re-expand it from the verified compressed template. After a start, the boot is stopped as a failed boot without adopting its disks and needs manual recovery. +Guest control readiness does not mean exposed bridge sockets have bound. The pinned +[SmolVM 1.34.1 guest](https://github.com/smol-machines/smolvm/blob/v1.34.1/crates/smolvm-agent/src/main.rs#L642) +starts [independent expose threads](https://github.com/smol-machines/smolvm/blob/v1.34.1/crates/smolvm-agent/src/publish_socket.rs#L57) +before signalling ready, without joining them. The pinned +[launcher removes old expose paths first](https://github.com/smol-machines/smolvm/blob/v1.34.1/src/agent/launcher.rs#L1955); +freshness depends on that ordering, because private socket metadata alone does not identify +its binder. A provider-pin upgrade must re-check that contract. Fresh boot waits up to 10 seconds +for all declared bridge sockets, before disk adoption and the full boot audit. Each poll +checks the retained provider identity before and after the socket observation. Missing +nodes and same-user, single-link sockets awaiting private mode can converge; a different +type, user, link count or unreadable identity refuses immediately. Only a complete private +socket set observed before the deadline succeeds. The wait never chmods, removes or connects +to sockets. Running-pool audits remain immediate. The existing failed-boot path stops only +the verified provider, records its absence and retains `failed-boot-stopped`, without +claiming a clean guest shutdown. Socket failures name the selected executor's +`runtime down --recover --json` on the same native home before retry; a project may not yet +have a mapping for ordinary `hack down`. The clock-driven real-socket regression reproduces +the ready-before-bind ordering without a VM, but cannot qualify actual provider scheduling. +HACK-2175's live gate is ten consecutive Mac Ctrl-C scenarios including pool stop/start. +The wait adds no signal handler: interruption retains the existing `booting` receipt +and stop/recovery rules, so that live gate also qualifies the extended startup window. + A start interrupted before the pool records its provider process (for example, `up` is killed) leaves the pool `creating` or `booting`, and `up` refuses. Each start clears the previous provider's already-dead identity before `booting`, so this covers every start, not only the diff --git a/packages/runtime-core/src/provider/artifact.rs b/packages/runtime-core/src/provider/artifact.rs index 440c30874..df516cf83 100644 --- a/packages/runtime-core/src/provider/artifact.rs +++ b/packages/runtime-core/src/provider/artifact.rs @@ -13,6 +13,9 @@ use std::path::{Path, PathBuf}; use std::time::Duration; pub const ARCHIVE_SHA256: &str = "8ebda502e332c739fdfc13b059be982cca66da9253c5229b4586af02c92781d9"; +// Bridge freshness relies on the pinned launcher's unlink-before-guest-ready +// ordering for expose paths (src/agent/launcher.rs:1955-1972 at v1.34.1). +// A pin upgrade must re-check that contract as well as the archive digest. pub const VERSION: &str = "1.34.1"; const DIRECTORY: &str = "smolvm-1.34.1-darwin-arm64"; diff --git a/packages/runtime-core/src/provider/bridge.rs b/packages/runtime-core/src/provider/bridge.rs index 4eb07b0dc..02ff13c52 100644 --- a/packages/runtime-core/src/provider/bridge.rs +++ b/packages/runtime-core/src/provider/bridge.rs @@ -91,33 +91,119 @@ pub(super) fn verify_sockets( directory: &std::path::Path, intent: BridgeIntent, ) -> Result<(), CandidateError> { + if sockets_ready(directory, intent)? { + Ok(()) + } else { + Err(socket_identity()) + } +} + +fn socket_identity() -> CandidateError { + CandidateError::new( + "bridge_socket_identity", + "Expected a private owned bridge socket.", + ) +} + +fn private_socket(metadata: &std::fs::Metadata, uid: u32) -> Result { use std::os::unix::fs::{FileTypeExt, MetadataExt}; + if !metadata.file_type().is_socket() || metadata.uid() != uid || metadata.nlink() != 1 { + return Err(socket_identity()); + } + Ok(metadata.mode() & 0o077 == 0) +} + +/// Missing nodes and not-yet-private owned sockets may converge during boot. +/// A different type, owner, link count or unreadable identity is never pending. +fn sockets_ready( + directory: &std::path::Path, + intent: BridgeIntent, +) -> Result { + let mut ready = true; for path in intent.guest_paths() { let path = directory.join( std::path::Path::new(&path) .file_name() .expect("fixed socket name"), ); - let metadata = std::fs::symlink_metadata(path).map_err(|_| { + let metadata = match std::fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + ready = false; + continue; + } + Err(_) => return Err(socket_identity()), + }; + // SAFETY: geteuid has no preconditions and cannot fail. + ready &= private_socket(&metadata, unsafe { libc::geteuid() })?; + } + Ok(ready) +} + +const BOOT_SOCKET_WAIT: std::time::Duration = std::time::Duration::from_secs(10); +const BOOT_SOCKET_POLL: std::time::Duration = std::time::Duration::from_millis(50); + +fn startup_socket_timeout() -> CandidateError { + CandidateError::new( + "bridge_socket_timeout", + "Native bridge sockets did not become private and owned before the startup deadline. Run the selected native executor with --candidate-root set to this native home and runtime down --recover --json before retrying.", + ) +} + +/// Only fresh boot waits: SmolVM's guest-ready signal does not join its expose +/// threads. Caller holds the pool lease and rechecks its retained provider on +/// both sides of each complete socket observation. No socket is changed here. +/// Socket freshness also relies on the pinned launcher's removal of old expose +/// paths before guest-ready; see the upgrade contract at `artifact::VERSION`. +pub(super) fn wait_for_sockets( + directory: &std::path::Path, + intent: BridgeIntent, + verify_provider: impl FnMut() -> Result<(), CandidateError>, +) -> Result<(), CandidateError> { + wait_for_sockets_with( + directory, + intent, + verify_provider, + &super::process::MonotonicClock::new(), + BOOT_SOCKET_WAIT, + ) +} + +fn wait_for_sockets_with( + directory: &std::path::Path, + intent: BridgeIntent, + mut verify_provider: impl FnMut() -> Result<(), CandidateError>, + clock: &impl super::process::Clock, + budget: std::time::Duration, +) -> Result<(), CandidateError> { + let deadline = clock.now().saturating_add(budget); + loop { + if clock.now() >= deadline { + return Err(startup_socket_timeout()); + } + verify_provider()?; + let ready = sockets_ready(directory, intent).map_err(|_| { CandidateError::new( "bridge_socket_identity", - "Expected a private owned bridge socket.", + "Bridge socket identity is unsafe; startup was refused. Run the selected native executor with --candidate-root set to this native home and runtime down --recover --json before retrying.", ) })?; - if !metadata.file_type().is_socket() - || metadata.uid() != unsafe { libc::geteuid() } - || metadata.mode() & 0o077 != 0 - || metadata.nlink() != 1 - { - return Err(CandidateError::new( - "bridge_socket_identity", - "Expected a private owned bridge socket.", - )); + verify_provider()?; + let remaining = deadline.saturating_sub(clock.now()); + if remaining.is_zero() { + return Err(startup_socket_timeout()); } + if ready { + return Ok(()); + } + clock.sleep(remaining.min(BOOT_SOCKET_POLL)); } - Ok(()) } +#[cfg(test)] +#[path = "bridge_startup_tests.rs"] +mod startup_tests; + #[cfg(test)] mod tests { use super::*; diff --git a/packages/runtime-core/src/provider/bridge_startup_tests.rs b/packages/runtime-core/src/provider/bridge_startup_tests.rs new file mode 100644 index 000000000..9e83aa6e7 --- /dev/null +++ b/packages/runtime-core/src/provider/bridge_startup_tests.rs @@ -0,0 +1,244 @@ +//! Guest/control readiness and exposed-socket binding are independent events. +//! A fake monotonic clock schedules real socket nodes in either order, without a VM. +use super::*; +use crate::provider::process::Clock; +use std::{ + cell::{Cell, RefCell}, + fs, + os::unix::{ + fs::{MetadataExt, PermissionsExt, symlink}, + net::UnixListener, + }, + path::PathBuf, + time::Duration, +}; + +struct Fixture { + root: PathBuf, + listeners: RefCell>, +} +impl Fixture { + fn new() -> Self { + Self { + root: crate::unique_test_root(&std::env::temp_dir(), "bridge-wait"), + listeners: RefCell::new(Vec::new()), + } + } + fn path(&self, slot: u8) -> PathBuf { + self.root.join(format!("bridge-{slot:02}.sock")) + } + fn bind(&self, slot: u8, mode: u32) { + self.listeners + .borrow_mut() + .push(UnixListener::bind(self.path(slot)).unwrap()); + self.mode(slot, mode); + } + fn mode(&self, slot: u8, mode: u32) { + fs::set_permissions(self.path(slot), fs::Permissions::from_mode(mode)).unwrap(); + } +} +impl Drop for Fixture { + fn drop(&mut self) { + self.listeners.get_mut().clear(); + fs::remove_dir_all(&self.root).unwrap(); + } +} + +struct Scheduled { + elapsed: Cell, + step: RefCell, +} +impl Scheduled { + fn new(step: F) -> Self { + Self { + elapsed: Cell::new(Duration::ZERO), + step: RefCell::new(step), + } + } +} +impl Clock for Scheduled { + fn now(&self) -> Duration { + self.elapsed.get() + } + fn sleep(&self, duration: Duration) { + self.elapsed.set(self.elapsed.get() + duration); + (self.step.borrow_mut())(self.elapsed.get()); + } +} +const BUDGET: Duration = Duration::from_millis(200); + +#[test] +fn guest_ready_before_all_bridge_binds_waits_for_complete_private_identity() { + let fixture = Fixture::new(); + let intent = BridgeIntent::new(2).unwrap(); + // The one-shot audit reproduces the old ordering failure: the provider + // control channel is ready while its expose threads have not yet run. + assert_eq!( + verify_sockets(&fixture.root, intent).unwrap_err().code, + "bridge_socket_identity" + ); + let clock = Scheduled::new(|elapsed: Duration| match elapsed.as_millis() { + 50 => fixture.bind(0, 0o700), + 100 => fixture.bind(1, 0o777), + 150 => fixture.mode(1, 0o700), + _ => panic!("unexpected wait"), + }); + wait_for_sockets_with(&fixture.root, intent, || Ok(()), &clock, BUDGET).unwrap(); + assert_eq!(clock.now(), Duration::from_millis(150)); + verify_sockets(&fixture.root, intent).unwrap(); +} + +#[test] +fn private_sockets_ready_first_need_no_wait_but_still_verify_the_provider_twice() { + let fixture = Fixture::new(); + fixture.bind(0, 0o600); + let clock = Scheduled::new(|_: Duration| panic!("already ready")); + let checks = Cell::new(0); + wait_for_sockets_with( + &fixture.root, + BridgeIntent::new(1).unwrap(), + || { + checks.set(checks.get() + 1); + Ok(()) + }, + &clock, + BUDGET, + ) + .unwrap(); + assert_eq!(checks.get(), 2); + assert_eq!(clock.now(), Duration::ZERO); +} + +#[test] +fn missing_or_never_private_sockets_expire_without_changing_them_and_name_recovery() { + for present in [false, true] { + let fixture = Fixture::new(); + if present { + fixture.bind(0, 0o777); + } + let clock = Scheduled::new(|_: Duration| {}); + let failure = wait_for_sockets_with( + &fixture.root, + BridgeIntent::new(1).unwrap(), + || Ok(()), + &clock, + BUDGET, + ) + .unwrap_err(); + assert_eq!(failure.code, "bridge_socket_timeout"); + assert!(failure.message_fixed); + assert!(failure.message.contains("runtime down --recover --json")); + assert_eq!(clock.now(), BUDGET); + if present { + assert_eq!( + fs::symlink_metadata(fixture.path(0)).unwrap().mode() & 0o777, + 0o777 + ); + } else { + assert!(!fixture.path(0).exists()); + } + } +} + +#[test] +fn unsafe_type_alias_and_link_count_refuse_even_when_an_earlier_slot_is_missing() { + for kind in ["file", "symlink", "hardlink"] { + let fixture = Fixture::new(); + match kind { + "file" => fs::write(fixture.path(1), b"foreign").unwrap(), + "symlink" => symlink(fixture.root.join("outside"), fixture.path(1)).unwrap(), + "hardlink" => { + fixture.bind(1, 0o700); + fs::hard_link(fixture.path(1), fixture.root.join("another-link")).unwrap(); + } + _ => unreachable!(), + } + let before = fs::symlink_metadata(fixture.path(1)).unwrap(); + let clock = Scheduled::new(|_: Duration| panic!("unsafe identity must not be retried")); + let failure = wait_for_sockets_with( + &fixture.root, + BridgeIntent::new(2).unwrap(), + || Ok(()), + &clock, + BUDGET, + ) + .unwrap_err(); + assert_eq!(failure.code, "bridge_socket_identity"); + assert!(failure.message_fixed); + assert!(failure.message.contains("runtime down --recover --json")); + let after = fs::symlink_metadata(fixture.path(1)).unwrap(); + assert_eq!( + (before.dev(), before.ino(), before.mode(), before.nlink()), + (after.dev(), after.ino(), after.mode(), after.nlink()) + ); + assert_eq!(clock.now(), Duration::ZERO); + } +} + +#[test] +fn another_owner_is_not_pending_or_private() { + let fixture = Fixture::new(); + fixture.bind(0, 0o700); + let metadata = fs::symlink_metadata(fixture.path(0)).unwrap(); + assert!(private_socket(&metadata, metadata.uid()).unwrap()); + assert_eq!( + private_socket(&metadata, metadata.uid().wrapping_add(1)) + .unwrap_err() + .code, + "bridge_socket_identity" + ); +} + +#[test] +fn provider_loss_before_or_after_observation_is_not_readiness_or_retry_authority() { + for fail_on in [1, 2] { + let fixture = Fixture::new(); + fixture.bind(0, 0o700); + let checks = Cell::new(0); + let clock = Scheduled::new(|_: Duration| panic!("provider loss must not be retried")); + let failure = wait_for_sockets_with( + &fixture.root, + BridgeIntent::new(1).unwrap(), + || { + checks.set(checks.get() + 1); + if checks.get() == fail_on { + Err(CandidateError::new( + "process_identity_mismatch", + "Provider changed.", + )) + } else { + Ok(()) + } + }, + &clock, + BUDGET, + ) + .unwrap_err(); + assert_eq!(failure.code, "process_identity_mismatch"); + assert_eq!(checks.get(), fail_on); + assert_eq!(clock.now(), Duration::ZERO); + } +} + +#[test] +fn readiness_observed_after_the_deadline_never_counts_as_success() { + let fixture = Fixture::new(); + fixture.bind(0, 0o700); + let clock = Scheduled::new(|_: Duration| panic!("no sleep needed")); + let checks = Cell::new(0); + let failure = wait_for_sockets_with( + &fixture.root, + BridgeIntent::new(1).unwrap(), + || { + checks.set(checks.get() + 1); + if checks.get() == 2 { + clock.elapsed.set(BUDGET); + } + Ok(()) + }, + &clock, + BUDGET, + ) + .unwrap_err(); + assert_eq!(failure.code, "bridge_socket_timeout"); +} diff --git a/packages/runtime-core/src/provider/lifecycle.rs b/packages/runtime-core/src/provider/lifecycle.rs index 073953d10..4fc8aa14a 100644 --- a/packages/runtime-core/src/provider/lifecycle.rs +++ b/packages/runtime-core/src/provider/lifecycle.rs @@ -1707,6 +1707,14 @@ fn finish_boot( lock, super::disk_template::Stage::AfterFirstStart, )?; + // The provider's agent-ready signal can precede expose-thread socket binds. + // Wait before adoption, without weakening the immediate audit used by + // running-pool commands. Failure keeps the existing verified-stop path. + if let Some(bridge) = owner.application_bridge { + super::bridge::wait_for_sockets(&owner.real_data_dir(candidate)?, bridge, || { + verify_live(candidate, owner) + })?; + } audit_boot(candidate, owner)?; let dir = owner.real_data_dir(candidate)?; let storage = identity::disk(&dir.join("storage.raw"))?; diff --git a/tests/native-project-start.test.ts b/tests/native-project-start.test.ts index c2e0d539e..3b3a3abb2 100644 --- a/tests/native-project-start.test.ts +++ b/tests/native-project-start.test.ts @@ -1,5 +1,6 @@ import { afterEach, expect, test } from "bun:test"; import { + chmod, mkdir, mkdtemp, readdir, @@ -59,6 +60,7 @@ import { verifyHttpsRoutes, } from "../src/backends/native-project-start.ts"; import { + invokeNativeRuntime, NativeRuntimeRequestError, nativeRuntimeFailure, } from "../src/backends/native-runtime-client.ts"; @@ -420,6 +422,38 @@ test("foreground saves after readiness and retains mapping after confirmed stop" "cleanup", ]); }); + +test("a pool bridge boot failure reaches foreground up with recovery guidance before any project mapping", async () => { + const { opts, events } = await fixture(false); + const message = + "Native bridge sockets did not become private and owned before the startup deadline. Run the selected native executor with --candidate-root set to this native home and runtime down --recover --json before retrying."; + await writeFile( + opts.runtime.binary, + `#!/usr/bin/env bun +console.error(JSON.stringify({ code: "bridge_socket_timeout", message: ${JSON.stringify(message)}, message_fixed: true, private_detail: "synthetic-secret" })); +process.exit(2); +` + ); + await chmod(opts.runtime.binary, 0o755); + const invoke = opts.dependencies.invoke; + opts.dependencies.invoke = async (request) => { + if (request.args[0] === "runtime" && request.args[1] === "up") { + return await invokeNativeRuntime(request); + } + return await invoke?.(request); + }; + const failure: unknown = await startNativeProject(opts).catch( + (error: unknown) => error + ); + expect(failure).toBeInstanceOf(ClosedCliError); + expect(String(failure)).toContain("runtime down --recover --json"); + expect(String(failure)).toContain("--candidate-root"); + expect(String(failure)).not.toContain("synthetic-secret"); + expect(nativeRuntimeFailure(failure)?.code).toBe("bridge_socket_timeout"); + expect(events).not.toContain("save"); + expect(events).not.toContain("ready"); + expect(events).toContain("cleanup"); +}); test("foreground reports value-free digests of the selected env after publishing the mapping", async () => { const { opts, events } = await fixture(); const seen: { run: string; fingerprint: ComposeEnvFingerprintRecord }[] = []; diff --git a/tests/native-runtime-client.test.ts b/tests/native-runtime-client.test.ts index c68a2e21b..3576d0a56 100644 --- a/tests/native-runtime-client.test.ts +++ b/tests/native-runtime-client.test.ts @@ -149,6 +149,7 @@ async function fixture() { if (process.argv.includes("hang")) await Bun.sleep(60_000); if (process.argv.includes("old-provider")) { console.error(JSON.stringify({code:"provider_pin_changed",message:"synthetic-secret-diagnostic"})); process.exit(2); } if (process.argv.includes("fixed-message")) { console.error(JSON.stringify({code:"graph_image",message:"Pinned image architecture or implicit volumes are incompatible.",message_fixed:true,service:"web"})); process.exit(2); } +if (process.argv.includes("bridge-socket-failure")) { console.error(JSON.stringify({code:"bridge_socket_timeout",message:"Native bridge sockets did not become private and owned before the startup deadline. Run the selected native executor with --candidate-root set to this native home and runtime down --recover --json before retrying.",message_fixed:true})); process.exit(2); } if (process.argv.includes("unsafe-fixed")) { console.error(JSON.stringify({code:"graph_image",message:"synthetic-secret-diagnostic\\nsecond line",message_fixed:true,service:"../synthetic-secret-diagnostic"})); process.exit(2); } if (process.argv.includes("plan-change")) { console.error(JSON.stringify({code:"graph_plan_reconcile",message:"synthetic-secret-diagnostic",plan_change:{version:1,service:"deps",field:"dependency_cache_volume"}})); process.exit(23); } if (process.argv.includes("plan-change-unsafe")) { console.error(JSON.stringify({code:"graph_hostname_change",message:"synthetic-secret-diagnostic",plan_change:{version:1,service:"synthetic secret diagnostic",field:"hostnames"}})); process.exit(23); } @@ -1000,6 +1001,19 @@ test("a runtime message marked fixed is shown with its validated service", async expect(String(unsafe)).not.toContain("synthetic-secret-diagnostic"); }); +test("a failed bridge boot names its one recovery command through the runtime client", async () => { + const runtime = await fixture(); + const error: unknown = await invokeNativeRuntime({ + runtime, + cwd: runtime.home, + args: ["bridge-socket-failure"], + }).catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(NativeRuntimeRequestError); + expect(error).toHaveProperty("nativeCode", "bridge_socket_timeout"); + expect(String(error)).toContain("runtime down --recover --json"); + expect(String(error)).not.toContain("nothing was started"); +}); + test("structured failures expose only a bounded code, never their message", async () => { const runtime = await fixture(); await expect( From 1457c63723e180e9fe6c6cfee211e8b8c249a864 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:18:03 +0000 Subject: [PATCH 7/9] fix(native): allow slow inferred route startup with clear diagnostics --- docs/guides/native-candidate.md | 24 +++- docs/reference/v5-model-coverage.md | 2 +- packages/runtime-core/build.rs | 15 +++ packages/runtime-core/guest/http-probe.c | 21 +++- packages/runtime-core/src/error.rs | 19 ++- .../runtime-core/src/project/execution.rs | 44 +++++-- .../src/project/route_probe_clock_tests.rs | 114 ++++++++++++++++++ .../runtime-core/src/provider/graph/mod.rs | 9 ++ .../runtime-core/src/provider/graph/probes.rs | 78 +++++++++++- .../src/provider/graph/startup/runtime.rs | 24 ++-- .../runtime-core/src/provider/http_probe.rs | 2 +- .../tests/fixtures/http-probe-clock.c | 30 +++++ src/backends/native-http-probe-admission.ts | 2 +- src/backends/native-runtime-client.ts | 39 +++++- tests/native-preflight-cli.test.ts | 2 +- tests/native-project-routing.test.ts | 3 +- tests/native-project-start.test.ts | 2 +- tests/native-runtime-client.test.ts | 49 ++++++++ 18 files changed, 431 insertions(+), 48 deletions(-) create mode 100644 packages/runtime-core/src/project/route_probe_clock_tests.rs create mode 100644 packages/runtime-core/tests/fixtures/http-probe-clock.c diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index 73deae714..8e1659535 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -2862,13 +2862,25 @@ shared-source mode; active project bind mounts still use the explicit project sh For a complete Caddy route with no authored `healthcheck`, native reviews a default probe: `GET /` on the route port, a 1000 ms timeout, three consecutive -failures, and a 60000 ms initial startup grace. It checks every second until its +failures, and a 240000 ms initial startup grace. It checks every second until its first success, then every 30 seconds for the rest of that probe generation. -After the grace, three failed startup checks make the service unhealthy before -the usual 300-second graph deadline, so startup names the failed service. -For a missing listener or HTTP 5xx during boot, check `hack logs`, fix the route -port, or add an explicit `healthcheck.x-hack-http` with your app's startup budget -and readiness path. +This allows slow first-page compilation, including a server first answering after +120 seconds. With a full timeout on each request, the grace plus three failed +startup samples takes at most about 246 seconds of probe time, leaving about +54 seconds of the usual 300-second graph budget for earlier startup work. +An inferred probe observed unhealthy names its service and route port. For a +missing listener or HTTP 5xx during boot, check `hack logs ` in the same +project/branch, fix the route +port, or declare `healthcheck.x-hack-http` with a longer `start_period_ms` and a +readiness path that answers 2xx. + +The 300-second deadline belongs to the whole graph; it is not renewed when a +service starts. Slow earlier services or `service_healthy` dependencies consume +it too. A routed service started after a dependency taking 240 seconds can reach +the graph deadline before its own probe finishes. That path still returns the +generic `graph_timeout`, without naming the waiting services. Use `hack logs` +and inspect the dependencies as well as the route. An explicit probe with a longer +grace does not extend the whole-graph deadline. A successful TCP connection and complete HTTP/1.0 or HTTP/1.1 response headers are required. HTTP 200–499, including `/` returning 404, is ready; 5xx, EOF, non-HTTP bytes, a timeout or partial headers are not. Redirects are not followed. At most diff --git a/docs/reference/v5-model-coverage.md b/docs/reference/v5-model-coverage.md index 0a3ef2029..54a1671d5 100644 --- a/docs/reference/v5-model-coverage.md +++ b/docs/reference/v5-model-coverage.md @@ -49,7 +49,7 @@ must be attached to the eventual exact integration revision. | Settled retained Compose route-attempt retirement; pinned references; interrupted deletion | **Coverage gap:** filesystem journal compaction is not modeled; native-route-stop covers a different stop protocol | Instance mutation lock and exact surviving claim witnesses in [journal compaction](../../src/lib/native-compose-route-claims.ts); [claim regressions](../../tests/native-compose-route-claims.test.ts) and [retained lifecycle integration](../../tests/native-compose-adoption-routing-generation.test.ts) | 24 successful operations keep two attempt directories and constant Linux descriptors; four integrated lifecycle operations check directory counts. Injected unlink failures cover same-store and fresh-store cleanup retries. Assumes cooperative same-user writers, the caller's mutation lock and working filesystem synchronization. No process-kill/power-loss durability proof or bound on unresolved/over-budget histories. | | Graph cleanup container stop batch: bounded in-flight stops, shared worker queue, connect-stage retry, batch-wide unreachable state, deadline expiry, worker panic | **Coverage gap:** no maintained model covers scheduling inside the stop batch; native-route-stop covers a different stop protocol | `dispatch` and `attempt` in [the stop batch](../../packages/runtime-core/src/provider/engine/stop.rs): at most 4 stops in flight, every worker joined; only `connect`/`connect_timeout` failures retry (25 ms doubling to 500 ms, within the batch deadline and 30 s of a stop's first connect failure); once one stop exhausts its window, later connect failures return at once; a deadline reached during backoff keeps the connect stage; `response`, `timeout` and `transport` never retry. Regressions in the same file's `provider::engine::stop` tests (bounded in-flight socket server, late socket, missing socket, 14-stop retry counts, persistent connect failures, oversleeping backoff, non-retried stages, worker panic) and `fourteen_service_connect_failures_keep_resumable_progress_and_retry_only_unstopped` | Tests cover exact attempt counts and the in-flight bound for fixed interleavings; no model explores all worker schedules or proves at most one delivered request per stop. Assumes a failed connect sent no request bytes and that the stop budget covers fully serialized grace periods. Graph-level resume after a partial batch is covered by tests only. Host qualification of the descriptor-pressure root cause (HACK-1302) remains open. | | Concurrent stop/publication; route claim and relay lifetime | native-route-stop, shared-https-lifetime, previous-boot-shared-https, relay-lifecycle-barrier, relay-authorization, relay-staging, relay-fence | Fenced route removal, relay owner, ready/removed protocol, authenticated observations | Native DNS/TLS/browser behavior; old-boot uncertainty and external routing. | -| Inferred Caddy route readiness without authored probes | Existing probe generation, exec identity and freshness checks are reused; no new lifecycle transition | `http_probe.rs` and compiled `guest/http-probe.c` tests cover TCP refusal, complete HTTP 2xx–4xx, 5xx, EOF, timeout, partial/oversized headers and bounded interim responses. Compiled-helper startup regressions cover no listener and boot-time 5xx reaching a named service failure before the graph deadline; a cadence regression covers the switch after first success. Bridge reservation regressions preserve running relay assignments when ready endpoints disappear. Project contracts preserve explicit checks and refuse inferred service_healthy dependencies; TypeScript route/preflight tests require the exact reviewed inferred policy. HTTPS tests require a fresh upstream response challenge for inferred routes | 32 route slots, 1 s startup/30 s steady interval, 1 s deadline, 8 KiB headers, four interim responses, three failed samples and 60 s initial grace. Bridge registry tests do not exercise continued route traffic; that remains a real-Caddy/VM check. No model explores HTTP parsing or Caddy handler behavior. TLS fixtures simulate the proxy marker; an actual Caddy/authority/VM run must qualify application 404 versus authority denial, SNI mismatch and missing upstream. | +| Inferred Caddy route readiness without authored probes | Existing probe generation, exec identity and freshness checks are reused; no new lifecycle transition | `http_probe.rs` and compiled `guest/http-probe.c` tests cover TCP refusal, complete HTTP 2xx–4xx, 5xx, EOF, timeout, partial/oversized headers and bounded interim responses. Compiled-helper startup regressions cover no listener and boot-time 5xx reaching a named service failure; route diagnostics name the port and remedies. A compiled production-transition fake-clock test drives graph readiness at 120 s, with the old 60 s grace as a negative control; it injects request outcomes and does not test real elapsed scheduling. A cadence regression covers the switch after first success. Bridge reservation regressions preserve running relay assignments when ready endpoints disappear. Project contracts preserve explicit checks and refuse inferred service_healthy dependencies; TypeScript route/preflight tests require the exact reviewed inferred policy. HTTPS tests require a fresh upstream response challenge for inferred routes | 32 route slots, 1 s startup/30 s steady interval, 1 s deadline, 8 KiB headers, four interim responses, three failed samples and 240 s initial grace. The separate 300 s graph budget includes earlier services and can still return unnamed graph_timeout. Bridge registry tests do not exercise continued route traffic; that remains a real-Caddy/VM check. No model explores HTTP parsing or Caddy handler behavior. TLS fixtures simulate the proxy marker; an actual Caddy/authority/VM run must qualify application 404 versus authority denial, SNI mismatch and missing upstream. | | Shared host helper instance: overlapping ups, session adoption, stale create decisions, replacement before finalize, epoch-record failure, interrupted cleanup, concurrent down, displaced launches, failed down session stop | shared-helper-instance | `src/lib/project-shared-helpers.ts` (shutdown generation, `assertCurrent`, `SharedHelperSessionOwnership`); `src/lib/project-lifecycle-controller.ts` session decision, the locked creation transaction (revalidate, publish, create, environment, record, stop on failure, all under one lock), launches fenced by the lock and a token check, abort and signal cleanup through the same epoch decision (`runIfOwned`), an unknown exact-presence answer failing the inspection, and the locked conditional finalize; `stopSessionBeforeReleasingHelpers` (down, restart, native retirement keep holders when the session stop fails); `runMuxCommand` bounds every lifecycle and backend mux call so it settles before the lock is released, and reports a timeout as unknown (never absence); `tests/mux-command-timeout.test.ts`; `tests/project-lifecycle-controller-ownership.test.ts`, `tests/project-shared-helpers.test.ts`, `tests/native-project-down.test.ts`, e2e `shared-host-helpers` | Helper holder/rollback and claim protocol are not modeled (claim fixtures cover them); timeout settlement is an assumption the code does not guarantee (a killed tmux client's queued command may still apply, so a late kill or window command can reach a newer session; consequences and the session-ID fence, not in this slice, are in the README); zellij is refused for shared-helper lifecycle sessions (token written after creation); the gap between a down's session stop and its release (an up that captured after the stopping mark can still succeed there) is a documented first-slice limitation; real mux/process/lock-file behavior. | | Engine restart policies: stop versus pending restart, guest engine restart, stale incarnations, terminal proof | service-restart-policy | Receipt-admitted policy, disarm-before-stop and `restarting` stop preflight in `shutdown.rs`; admission refusals for incarnation-bound delivery classes in `restart_policy.rs`; owner-driven `Rebind` after full retirement, fresh ownership/attachment checks and a new reservation in `bridges/restart_rebind.rs`; route probe sample-generation rearm; Rust tests for admission, endpoint identity, port-busy refusal, disarm and stop preflight; portable selected-child readiness guard plus macOS two-real-child polling regression prevent historical sibling exits from rejecting a replacement or misclassifying its own listener | Real guest engine behavior (crash restart, stop during backoff, disarm before deletion) needs native host evidence; several services, selected-child readiness during sibling failure, backoff timing and retry counting are not modeled. Same-port/route reachability after a policy restart, route probe rearm and per-slot child retirement still require real-VM host evidence. Rebind abstracts retirement/reservation/publication; relay lifecycle, fence and staging models cover their separate failure boundaries. | | Foreground Ctrl-C down hooks: concurrent `down`, second interrupt, crash between or during hooks, stop during `up.after`, next generation | native-foreground-down-hooks | Value-free compare-and-swap journal in [down hooks](../../src/backends/native-project-down-hooks.ts), foreground orchestration in [foreground stop](../../src/backends/native-project-foreground-stop.ts) and `hack down` in [native down](../../src/backends/native-project-down.ts); [journal](../../tests/native-project-down-hooks.test.ts), [foreground](../../tests/native-project-start-stop-hooks.test.ts) and [down](../../tests/native-project-down-journal.test.ts) regressions | Durable writes are atomic in the model; real fsync/power loss, a stale journal lock after a crash inside the lock, and the live process-group behavior of the graph owner on a host are not covered. One generation and at most two `down` invocations are explored. | diff --git a/packages/runtime-core/build.rs b/packages/runtime-core/build.rs index d97bfe0a9..1a1d15429 100644 --- a/packages/runtime-core/build.rs +++ b/packages/runtime-core/build.rs @@ -193,6 +193,21 @@ fn build_http_probe() { .status() .expect("build host HTTP probe test driver"); assert!(result.success(), "host HTTP probe build failed"); + println!("cargo:rerun-if-changed=tests/fixtures/http-probe-clock.c"); + let result = Command::new("cc") + .args([ + "-std=c11", + "-O2", + "-Wall", + "-Wextra", + "-Werror", + "tests/fixtures/http-probe-clock.c", + "-o", + ]) + .arg(out.join("http-probe-clock-host")) + .status() + .expect("build fake-clock probe test driver"); + assert!(result.success(), "fake-clock HTTP probe build failed"); } fn build_loopback_identity() { diff --git a/packages/runtime-core/guest/http-probe.c b/packages/runtime-core/guest/http-probe.c index 5d27f754b..468850432 100644 --- a/packages/runtime-core/guest/http-probe.c +++ b/packages/runtime-core/guest/http-probe.c @@ -124,6 +124,19 @@ static bool publish(int directory, const char *name, const char *generation, uin // Ephemeral status is atomically replaced, never a durable readiness receipt. return ok && !renameat(directory, "status.pending", directory, name); } +// Shared with the fake-clock harness: only success ends grace early. The clock +// and request outcome come from the resident loop, not from retained samples. +static void sample_health(bool success, uint64_t elapsed, uint64_t grace, uint64_t retries, + uint64_t *failures, int *health, bool *ever_healthy) { + if (success) { *failures = 0; *health = 1; *ever_healthy = true; } + else if (*ever_healthy || elapsed >= grace) { + if (*failures < retries) ++*failures; + if (*failures >= retries) *health = 0; + } +} +static uint64_t sample_period(bool route, bool ever_healthy, uint64_t interval) { + return route && !ever_healthy && interval > 1000 ? 1000 : interval; +} int main(int argc, char **argv) { if (argc != 9 && argc != 10) return 125; bool route = argc == 10; @@ -164,11 +177,7 @@ int main(int argc, char **argv) { bool success = request(port, argv[2], timeout, route); if (stopping) break; uint64_t finished = millis(CLOCK_MONOTONIC); - if (success) { failures = 0; health = 1; ever_healthy = true; } - else if (ever_healthy || finished - start >= grace) { - if (failures < retries) failures++; - if (failures >= retries) health = 0; - } + sample_health(success, finished - start, grace, retries, &failures, &health, &ever_healthy); ++sequence; // One bounded diagnostic survives later success; it never supplies current readiness. // On the first request only, health == 1 means success even during startup grace. @@ -177,7 +186,7 @@ int main(int argc, char **argv) { // Generated route probes check quickly during initial compilation, then // use the reviewed steady interval for the rest of this exec generation. // An unhealthy sample never re-enables the fast startup cadence. - uint64_t period = route && !ever_healthy && interval > 1000 ? 1000 : interval; + uint64_t period = sample_period(route, ever_healthy, interval); uint64_t next = begun + period; while (!stopping && millis(CLOCK_MONOTONIC) < next) { uint64_t delay = next - millis(CLOCK_MONOTONIC); diff --git a/packages/runtime-core/src/error.rs b/packages/runtime-core/src/error.rs index 7ddf9b888..1272750b5 100644 --- a/packages/runtime-core/src/error.rs +++ b/packages/runtime-core/src/error.rs @@ -130,9 +130,18 @@ pub struct PlanChangeDiagnostic { pub field: PlanChangeField, } +/// Only the public reviewed port of an unhealthy inferred probe. No path, +/// response, other configuration or application output crosses this boundary. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct RouteProbeDiagnostic { + pub version: u8, + pub port: u16, +} + /// Value-free refusal locations, boxed together so every /// `Result<_, CandidateError>` stays small; serialized flat as -/// `process_refusals` and `plan_change`. +/// `process_refusals`, `plan_change` and `route_probe`. #[derive(Debug, Default, Serialize)] pub struct RefusalDiagnostics { /// Every service whose selected process a private wrapper refused, found in @@ -142,6 +151,8 @@ pub struct RefusalDiagnostics { /// Value-free location of a refused retained-run edit; see [`PlanChangeDiagnostic`]. #[serde(skip_serializing_if = "Option::is_none")] pub plan_change: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub route_probe: Option, } /// Closed diagnostic vocabulary for the read-only log follower. No stream bytes. @@ -232,6 +243,12 @@ impl CandidateError { self } + pub fn with_route_probe(mut self, port: u16) -> Self { + self.refusals.get_or_insert_default().route_probe = + Some(RouteProbeDiagnostic { version: 1, port }); + self + } + /// Name every service and field a private process wrapper refused. pub fn with_process_refusals(mut self, refusals: Vec) -> Self { self.refusals.get_or_insert_default().process_refusals = Some(ProcessRefusalsDiagnostic { diff --git a/packages/runtime-core/src/project/execution.rs b/packages/runtime-core/src/project/execution.rs index a361ce31b..b6d57daf0 100644 --- a/packages/runtime-core/src/project/execution.rs +++ b/packages/runtime-core/src/project/execution.rs @@ -101,6 +101,12 @@ pub trait Driver { fn record(&mut self, event: Event<'_>) -> Result<(), CandidateError>; fn start(&mut self, service: &str) -> Result<(), CandidateError>; fn observe(&mut self, service: &str) -> Result; + + /// Add reviewed readiness context to an observed failure, without copying + /// application output or private configuration into the error. + fn service_failure(&self, service: &str, state: Observation) -> CandidateError { + failed_service_error("graph_service_failed", service, state) + } } fn error(code: &'static str, message: impl Into) -> CandidateError { CandidateError::new(code, message) @@ -266,7 +272,7 @@ fn inspect( observations.insert(name.into(), state); } if state.failed() { - return Err(failed_service_error("graph_service_failed", name, state)); + return Err(driver.service_failure(name, state)); } Ok(state) } @@ -293,8 +299,8 @@ pub(crate) fn failed_service_error( ) .with_service(name) } -fn before_deadline(deadline: Instant) -> Result<(), CandidateError> { - if Instant::now() >= deadline { +fn before_deadline(deadline: Instant, now: Instant) -> Result<(), CandidateError> { + if now >= deadline { return Err(error( "graph_timeout", "Graph readiness deadline exceeded; owned state is retained.", @@ -313,10 +319,20 @@ pub fn run( } fn run_with_wait( + graph: &Graph, + driver: &mut impl Driver, + timeout: Duration, + wait: impl FnMut(Duration), +) -> Result<(), CandidateError> { + run_with_clock(graph, driver, timeout, wait, Instant::now) +} + +fn run_with_clock( graph: &Graph, driver: &mut impl Driver, timeout: Duration, mut wait: impl FnMut(Duration), + mut now: impl FnMut() -> Instant, ) -> Result<(), CandidateError> { let order = graph.execution_order()?; if timeout.is_zero() || timeout > Duration::from_secs(600) { @@ -325,15 +341,15 @@ fn run_with_wait( "Graph timeout must be greater than zero and at most 600 seconds.", )); } - let deadline = Instant::now() + timeout; + let deadline = now() + timeout; let mut started = BTreeSet::::new(); let mut observations = BTreeMap::new(); loop { - before_deadline(deadline)?; + before_deadline(deadline, now())?; driver.check_cancelled()?; for name in &started { inspect(driver, name, &mut observations)?; - before_deadline(deadline)?; + before_deadline(deadline, now())?; driver.check_cancelled()?; } for name in &order { @@ -349,7 +365,7 @@ fn run_with_wait( break; } let state = inspect(driver, dependency, &mut observations)?; - before_deadline(deadline)?; + before_deadline(deadline, now())?; driver.check_cancelled()?; if !state.satisfies(*condition) { ready = false; @@ -359,10 +375,10 @@ fn run_with_wait( if !ready { continue; } - before_deadline(deadline)?; + before_deadline(deadline, now())?; driver.check_cancelled()?; driver.record(Event::StartIntent { service: name })?; - before_deadline(deadline)?; + before_deadline(deadline, now())?; driver.check_cancelled()?; if let Err(failure) = driver.start(name) { driver.record(Event::StartUncertain { service: name })?; @@ -371,14 +387,14 @@ fn run_with_wait( driver.record(Event::Started { service: name })?; started.insert(name.clone()); inspect(driver, name, &mut observations)?; - before_deadline(deadline)?; + before_deadline(deadline, now())?; driver.check_cancelled()?; } if started.len() == graph.services.len() { let mut ready = true; for (name, service) in &graph.services { let state = inspect(driver, name, &mut observations)?; - before_deadline(deadline)?; + before_deadline(deadline, now())?; driver.check_cancelled()?; ready &= state.satisfies(service.ready); } @@ -387,10 +403,14 @@ fn run_with_wait( return Ok(()); } } - wait(Duration::from_millis(100).min(deadline.saturating_duration_since(Instant::now()))); + wait(Duration::from_millis(100).min(deadline.saturating_duration_since(now()))); } } +#[cfg(all(test, feature = "native-http-probe"))] +#[path = "route_probe_clock_tests.rs"] +mod route_probe_clock_tests; + #[cfg(test)] mod tests { use super::*; diff --git a/packages/runtime-core/src/project/route_probe_clock_tests.rs b/packages/runtime-core/src/project/route_probe_clock_tests.rs new file mode 100644 index 000000000..a6d1a5c59 --- /dev/null +++ b/packages/runtime-core/src/project/route_probe_clock_tests.rs @@ -0,0 +1,114 @@ +//! Fake time drives both the production probe transition and graph deadline. +//! Socket/HTTP parsing, live exec identity and publication are tested separately. +use super::*; +use crate::provider::http_probe::HttpProbe; +use std::{cell::Cell, process::Command}; + +struct Probed<'a> { + clock: &'a Cell, + config: HttpProbe, + samples: Vec<(u64, u64, u64)>, + started: bool, + ready: bool, +} +impl Driver for Probed<'_> { + fn record(&mut self, event: Event<'_>) -> Result<(), CandidateError> { + if matches!(event, Event::Ready) { + self.ready = true; + } + Ok(()) + } + fn start(&mut self, service: &str) -> Result<(), CandidateError> { + assert_eq!(service, "web"); + assert!(!self.started, "a service must never be replayed"); + self.started = true; + Ok(()) + } + fn observe(&mut self, _: &str) -> Result { + let elapsed = self.clock.get().as_millis() as u64; + let (sequence, &(observed, health, failures)) = self + .samples + .iter() + .enumerate() + .rev() + .find(|(_, (time, _, _))| *time <= elapsed) + .unwrap(); + let generation = "0".repeat(32); + let raw = format!( + "v1 {generation} {sequence} {health} {failures} {} 0\n", + observed + 100_000 + ); + Ok(Observation::Running { + health: self + .config + .health(&raw, &generation, true, elapsed + 100_000)?, + }) + } +} + +#[test] +fn a_route_first_answering_at_120_seconds_starts_under_the_graph_budget() { + let graph = Graph::from_services(BTreeMap::from([( + "web".into(), + Service { + dependencies: BTreeMap::new(), + ready: Condition::Healthy, + }, + )])) + .unwrap(); + // The old grace is a negative control: the same delayed success must fail + // early with it. The reviewed default must reach graph readiness at 120 s. + for grace in [60_000, HttpProbe::inferred_route(3000).start_period_ms] { + let mut config = HttpProbe::inferred_route(3000); + config.start_period_ms = grace; + let output = Command::new(concat!(env!("OUT_DIR"), "/http-probe-clock-host")) + .args([ + grace.to_string(), + config.interval_ms.to_string(), + config.timeout_ms.to_string(), + config.retries.to_string(), + "120000".into(), + ]) + .output() + .unwrap(); + assert!(output.status.success()); + assert!(output.stderr.is_empty()); + let samples = String::from_utf8(output.stdout) + .unwrap() + .lines() + .map(|line| { + let fields: Vec = line + .split(' ') + .map(|field| field.parse().unwrap()) + .collect(); + (fields[0], fields[1], fields[2]) + }) + .collect(); + let clock = Cell::new(Duration::ZERO); + let base = Instant::now(); + let mut driver = Probed { + clock: &clock, + config, + samples, + started: false, + ready: false, + }; + let result = run_with_clock( + &graph, + &mut driver, + Duration::from_secs(300), + |delay| clock.set(clock.get() + delay), + || base + clock.get(), + ); + assert!(driver.started); + if grace == 60_000 { + assert_eq!(result.unwrap_err().code, "graph_service_failed"); + assert!(!driver.ready); + assert!(clock.get() < Duration::from_secs(120)); + } else { + result.unwrap(); + assert!(driver.ready); + assert_eq!(clock.get(), Duration::from_secs(120)); + } + } +} diff --git a/packages/runtime-core/src/provider/graph/mod.rs b/packages/runtime-core/src/provider/graph/mod.rs index 180f4f777..b5106fbb5 100644 --- a/packages/runtime-core/src/provider/graph/mod.rs +++ b/packages/runtime-core/src/provider/graph/mod.rs @@ -992,6 +992,15 @@ impl Session<'_, '_> { } } impl Driver for Session<'_, '_> { + fn service_failure(&self, name: &str, observation: Observation) -> CandidateError { + probes::service_failure( + "graph_service_failed", + name, + observation, + self.receipt.probes.get(name).map(|probe| &probe.config), + ) + } + fn check_cancelled(&self) -> Result<(), CandidateError> { if let Some(startup) = &self.startup { startup.check_cancelled()?; diff --git a/packages/runtime-core/src/provider/graph/probes.rs b/packages/runtime-core/src/provider/graph/probes.rs index 7fc28b88f..a596f4a5b 100644 --- a/packages/runtime-core/src/provider/graph/probes.rs +++ b/packages/runtime-core/src/provider/graph/probes.rs @@ -1,4 +1,4 @@ -use super::super::http_probe::HttpProbe; +use super::super::http_probe::{HttpProbe, HttpProbePolicy}; use super::*; #[cfg(feature = "native-http-probe")] use { @@ -18,6 +18,82 @@ pub struct Probe { pub started_ms: u64, pub phase: String, } + +/// Only an inferred route's unhealthy observation gets route guidance. Explicit +/// healthchecks, exits and deaths keep their existing diagnostic semantics. +pub(super) fn service_failure( + code: &'static str, + name: &str, + observation: Observation, + probe: Option<&HttpProbe>, +) -> CandidateError { + if let Some(probe) = probe.filter(|probe| { + probe.policy == Some(HttpProbePolicy::Route) + && matches!( + observation, + Observation::Running { + health: execution::Health::Unhealthy + } + ) + }) { + return error( + code, + format!( + "Owned graph service {name} is unhealthy on inferred route port {}; no further services will start. Check hack logs {name} in the same project/branch, fix the route port, or declare healthcheck.x-hack-http with a longer start_period_ms and a readiness path, within the 300-second graph budget.", + probe.port + ), + ) + .with_service(name) + .with_route_probe(probe.port); + } + execution::failed_service_error(code, name, observation) +} + +#[cfg(test)] +mod route_failure_tests { + use super::*; + + #[test] + fn only_an_unhealthy_inferred_probe_names_its_route_and_remedies() { + let inferred = HttpProbe::inferred_route(3000); + let unhealthy = Observation::Running { + health: execution::Health::Unhealthy, + }; + for code in ["graph_service_failed", "graph_startup_application_failed"] { + let failure = service_failure(code, "web", unhealthy, Some(&inferred)); + assert_eq!(failure.code, code); + assert_eq!(failure.service.as_deref(), Some("web")); + assert!( + failure + .message + .contains("web is unhealthy on inferred route port 3000") + ); + assert!(failure.message.contains("hack logs web")); + assert!(failure.message.contains("healthcheck.x-hack-http")); + assert!(failure.message.contains("longer start_period_ms")); + let value = serde_json::to_value(failure).unwrap(); + assert_eq!(value["route_probe"], json!({"version":1,"port":3000})); + assert!(value.get("message_fixed").is_none()); + } + let mut explicit = inferred.clone(); + explicit.policy = None; + for (observation, probe) in [ + (unhealthy, None), + (unhealthy, Some(&explicit)), + (Observation::Dead, Some(&inferred)), + (Observation::Exited { code: 1 }, Some(&inferred)), + ] { + let failure = service_failure("graph_service_failed", "web", observation, probe); + assert!(!failure.message.contains("inferred route")); + assert!( + serde_json::to_value(failure) + .unwrap() + .get("route_probe") + .is_none() + ); + } + } +} pub(super) fn token() -> Result { use std::io::Read; let mut bytes = [0u8; 16]; diff --git a/packages/runtime-core/src/provider/graph/startup/runtime.rs b/packages/runtime-core/src/provider/graph/startup/runtime.rs index c6bc69c09..92f93650d 100644 --- a/packages/runtime-core/src/provider/graph/startup/runtime.rs +++ b/packages/runtime-core/src/provider/graph/startup/runtime.rs @@ -121,12 +121,12 @@ fn stage_refused(code: &'static str) -> CandidateError { /// An owned service failed while dependency relays were attached. The stage code /// stays `graph_startup_application_failed`; the message and service name the /// failed service (exit code, death or health) instead of a validation failure. -fn application_failed(name: &str, observation: Observation) -> CandidateError { - crate::project::execution::failed_service_error( - "graph_startup_application_failed", - name, - observation, - ) +fn application_failed( + name: &str, + observation: Observation, + probe: Option<&crate::provider::http_probe::HttpProbe>, +) -> CandidateError { + probes::service_failure("graph_startup_application_failed", name, observation, probe) } // Fresh and restored graphs both provision a new container generation. Retained // stopped/ready receipts cannot authorize listener attachment. @@ -864,7 +864,11 @@ impl Driver for HostRelayRuntime { state::write(&root.join("state.json"), receipt)?; // Name the failed service, as a graph without dependency // bindings does, rather than a dependency validation failure. - return Err(application_failed(name, observation)); + return Err(application_failed( + name, + observation, + receipt.probes.get(name).map(|probe| &probe.config), + )); } if let Err(error) = verify_exited_listener(receipt.readiness.get(name), &observed) { if error.code == "graph_startup_listener_unexpected_exit" @@ -1651,17 +1655,17 @@ mod application_failure_tests { use super::*; #[test] fn an_application_failure_names_the_service_not_dependency_validation() { - let error = application_failed("deps", Observation::Exited { code: 3 }); + let error = application_failed("deps", Observation::Exited { code: 3 }, None); assert_eq!(error.code, "graph_startup_application_failed"); assert_eq!(error.service.as_deref(), Some("deps")); let message = error.message.to_string(); assert!(message.contains("deps exited with code 3"), "{message}"); assert!(!message.contains("validation"), "{message}"); - let dead = application_failed("deps", Observation::Dead); + let dead = application_failed("deps", Observation::Dead, None); assert!(dead.message.to_string().contains("dead or OOM-killed")); // An invalid service identifier is never echoed as the service. assert_eq!( - application_failed("bad name", Observation::Dead).service, + application_failed("bad name", Observation::Dead, None).service, None ); } diff --git a/packages/runtime-core/src/provider/http_probe.rs b/packages/runtime-core/src/provider/http_probe.rs index 725d048e9..375c75531 100644 --- a/packages/runtime-core/src/provider/http_probe.rs +++ b/packages/runtime-core/src/provider/http_probe.rs @@ -45,7 +45,7 @@ impl HttpProbe { interval_ms: 30_000, timeout_ms: 1_000, retries: 3, - start_period_ms: 60_000, + start_period_ms: 240_000, policy: Some(HttpProbePolicy::Route), } } diff --git a/packages/runtime-core/tests/fixtures/http-probe-clock.c b/packages/runtime-core/tests/fixtures/http-probe-clock.c new file mode 100644 index 000000000..44db26018 --- /dev/null +++ b/packages/runtime-core/tests/fixtures/http-probe-clock.c @@ -0,0 +1,30 @@ +// Exercise the production sample transition and cadence with a monotonic fake +// clock. Networking and publication are covered by http-probe-host regressions. +#define main resident_probe_main +#include "../../guest/http-probe.c" +#undef main + +int main(int argc, char **argv) { + if (argc != 6) return 125; + uint64_t grace = number(argv[1], 0, 3600000); + uint64_t interval = number(argv[2], 10, 3600000); + uint64_t timeout = number(argv[3], 1, 60000); + uint64_t retries = number(argv[4], 1, 100); + uint64_t ready_after = number(argv[5], 0, 3600000); + uint64_t now = 0, failures = 0; + int health = 2; + bool ever_healthy = false; + puts("0 2 0"); + while (now <= 300000) { + uint64_t begun = now; + bool success = begun >= ready_after; + // A failed sample consumes the full reviewed request timeout. + now += success ? 0 : timeout; + sample_health(success, now, grace, retries, &failures, &health, &ever_healthy); + printf("%llu %d %llu\n", (unsigned long long)now, health, + (unsigned long long)failures); + uint64_t next = begun + sample_period(true, ever_healthy, interval); + if (next > now) now = next; + } + return 0; +} diff --git a/src/backends/native-http-probe-admission.ts b/src/backends/native-http-probe-admission.ts index 76695d19c..86957b1ba 100644 --- a/src/backends/native-http-probe-admission.ts +++ b/src/backends/native-http-probe-admission.ts @@ -30,7 +30,7 @@ export function isInferredNativeRouteProbe( probe.interval_ms === 30_000 && probe.timeout_ms === 1000 && probe.retries === 3 && - probe.start_period_ms === 60_000 + probe.start_period_ms === 240_000 ); } function bounded(value: unknown, min: number, max: number): boolean { diff --git a/src/backends/native-runtime-client.ts b/src/backends/native-runtime-client.ts index 959b0ab2e..fd46de9aa 100644 --- a/src/backends/native-runtime-client.ts +++ b/src/backends/native-runtime-client.ts @@ -33,7 +33,7 @@ interface NativeFailure { readonly code: string; readonly logPhase?: NativeLogFailurePhase; readonly causeCode?: string; - /** Runtime message the runtime marked as a compile-time literal (value-free). */ + /** Fixed runtime text, or fixed guidance rendered from a validated diagnostic. */ readonly message?: string; /** Admitted service the runtime named, as a plain service identifier. */ readonly service?: string; @@ -76,16 +76,43 @@ function fixedNativeMessage( : undefined; } -/** The fixed message and validated service a runtime failure carries, if any. */ +/** Render only the closed inferred-route diagnostic; never runtime error text. */ +function routeProbeFailureMessage( + value: Record, + service: string | undefined +): string | undefined { + const probe = value.route_probe; + if ( + !( + service && + ["graph_service_failed", "graph_startup_application_failed"].includes( + typeof value.code === "string" ? value.code : "" + ) && + isRecord(probe) + ) || + Object.keys(probe).length !== 2 || + probe.version !== 1 || + typeof probe.port !== "number" || + !Number.isSafeInteger(probe.port) || + probe.port < 1 || + probe.port > 65_535 + ) { + return; + } + return `Inferred route probe on port ${probe.port} is unhealthy. Check hack logs ${service} in the same project/branch, fix the route port, or declare healthcheck.x-hack-http with a longer start_period_ms and a readiness path, within the 300-second graph budget.`; +} + +/** Fixed/closed diagnostic text and the validated service a failure carries. */ export function nativeFailureDetail(value: Record): { message?: string; service?: string; } { - const message = fixedNativeMessage(value); const service = typeof value.service === "string" && NATIVE_SERVICE.test(value.service) ? value.service : undefined; + const message = + routeProbeFailureMessage(value, service) ?? fixedNativeMessage(value); return { ...(message ? { message } : {}), ...(service ? { service } : {}), @@ -124,7 +151,7 @@ export function retainNativeRuntimeFailure(opts: { /** * Value-free runtime detail appended after a closed native failure sentence: - * the fixed runtime message and the service it concerns, when known. + * fixed runtime text or closed probe guidance, and its validated service. */ export function nativeFailureNote(detail: { readonly message?: string; @@ -148,7 +175,7 @@ export class NativeRuntimeRequestError extends ClosedCliError { readonly stopSignal: "SIGINT" | "SIGTERM" | undefined; readonly nativeCode: string | undefined; readonly nativeCauseCode: string | undefined; - /** Fixed (value-free) runtime message; see {@link nativeFailureNote}. */ + /** Fixed/closed diagnostic text; see {@link nativeFailureNote}. */ readonly nativeMessage: string | undefined; readonly nativeService: string | undefined; readonly nativeStopFailures: readonly NativeStopFailure[] | undefined; @@ -158,7 +185,7 @@ export class NativeRuntimeRequestError extends ClosedCliError { readonly message: string; readonly nativeCode?: string; readonly nativeCauseCode?: string; - /** Only fixed text admitted by nativeFailureDetail; this can cross IPC. */ + /** Only safe text admitted by nativeFailureDetail; this can cross IPC. */ readonly nativeMessage?: string; readonly nativeService?: string; readonly nativeStopFailures?: readonly NativeStopFailure[]; diff --git a/tests/native-preflight-cli.test.ts b/tests/native-preflight-cli.test.ts index 8833abd35..44dd53fff 100644 --- a/tests/native-preflight-cli.test.ts +++ b/tests/native-preflight-cli.test.ts @@ -825,7 +825,7 @@ test( interval_ms: 30_000, timeout_ms: 1000, retries: 3, - start_period_ms: 60_000, + start_period_ms: 240_000, }, }, }, diff --git a/tests/native-project-routing.test.ts b/tests/native-project-routing.test.ts index 0435d2510..d973f3654 100644 --- a/tests/native-project-routing.test.ts +++ b/tests/native-project-routing.test.ts @@ -171,7 +171,7 @@ test("a route without an authored healthcheck accepts only its exact reviewed in interval_ms: 30_000, timeout_ms: 1000, retries: 3, - start_period_ms: 60_000, + start_period_ms: 240_000, }; const opts = { plan: { @@ -208,6 +208,7 @@ test("a route without an authored healthcheck accepts only its exact reviewed in { ...inferred, path: "/health" }, { ...inferred, policy: "anything" }, { ...inferred, timeout_ms: 60_000 }, + { ...inferred, start_period_ms: 60_000 }, { ...inferred, extra: true }, { port: 3000 }, ]) { diff --git a/tests/native-project-start.test.ts b/tests/native-project-start.test.ts index 588d8fa35..18a080ac1 100644 --- a/tests/native-project-start.test.ts +++ b/tests/native-project-start.test.ts @@ -3165,7 +3165,7 @@ test("inferred HTTPS readiness accepts 2xx-4xx without following redirects and r interval_ms: 30_000, timeout_ms: 1000, retries: 3, - start_period_ms: 60_000, + start_period_ms: 240_000, }; const plan = { services: { diff --git a/tests/native-runtime-client.test.ts b/tests/native-runtime-client.test.ts index c68a2e21b..fddb8653b 100644 --- a/tests/native-runtime-client.test.ts +++ b/tests/native-runtime-client.test.ts @@ -8,6 +8,7 @@ import { NATIVE_EXIT_DRAIN_BOUND_MS, NativeRuntimeRequestError, nativeCodesReportRuntimeNotRunning, + nativeFailureDetail, resolveNativeRuntimeSelection, } from "../src/backends/native-runtime-client.ts"; import { ClosedCliError } from "../src/lib/cli-result.ts"; @@ -149,6 +150,7 @@ async function fixture() { if (process.argv.includes("hang")) await Bun.sleep(60_000); if (process.argv.includes("old-provider")) { console.error(JSON.stringify({code:"provider_pin_changed",message:"synthetic-secret-diagnostic"})); process.exit(2); } if (process.argv.includes("fixed-message")) { console.error(JSON.stringify({code:"graph_image",message:"Pinned image architecture or implicit volumes are incompatible.",message_fixed:true,service:"web"})); process.exit(2); } +if (process.argv.includes("route-probe-failure")) { console.error(JSON.stringify({code:"graph_service_failed",message:"synthetic-private-response",service:"web",route_probe:{version:1,port:3000}})); process.exit(2); } if (process.argv.includes("unsafe-fixed")) { console.error(JSON.stringify({code:"graph_image",message:"synthetic-secret-diagnostic\\nsecond line",message_fixed:true,service:"../synthetic-secret-diagnostic"})); process.exit(2); } if (process.argv.includes("plan-change")) { console.error(JSON.stringify({code:"graph_plan_reconcile",message:"synthetic-secret-diagnostic",plan_change:{version:1,service:"deps",field:"dependency_cache_volume"}})); process.exit(23); } if (process.argv.includes("plan-change-unsafe")) { console.error(JSON.stringify({code:"graph_hostname_change",message:"synthetic-secret-diagnostic",plan_change:{version:1,service:"synthetic secret diagnostic",field:"hostnames"}})); process.exit(23); } @@ -1009,6 +1011,53 @@ test("structured failures expose only a bounded code, never their message", asyn ); }); +test("inferred route failures render only validated service and port guidance", async () => { + const reply = { + code: "graph_service_failed", + service: "web", + route_probe: { version: 1, port: 3000 }, + message: "synthetic-private-response", + }; + const expected = { + service: "web", + message: + "Inferred route probe on port 3000 is unhealthy. Check hack logs web in the same project/branch, fix the route port, or declare healthcheck.x-hack-http with a longer start_period_ms and a readiness path, within the 300-second graph budget.", + }; + expect(nativeFailureDetail(reply)).toEqual(expected); + const longest = nativeFailureDetail({ ...reply, service: "w".repeat(128) }); + expect(longest.message?.length).toBeLessThanOrEqual(400); + const runtime = await fixture(); + const failure: unknown = await invokeNativeRuntime({ + runtime, + cwd: runtime.home, + args: ["route-probe-failure"], + }).catch((caught: unknown) => caught); + expect(failure).toBeInstanceOf(NativeRuntimeRequestError); + expect((failure as Error).message).toContain( + `Service web: ${expected.message}` + ); + expect((failure as Error).message).not.toContain( + "synthetic-private-response" + ); + expect( + nativeFailureDetail({ ...reply, code: "graph_startup_application_failed" }) + ).toEqual(expected); + for (const patch of [ + { code: "graph_timeout" }, + { route_probe: { version: 2, port: 3000 } }, + { route_probe: { version: 1, port: 0 } }, + { route_probe: { version: 1, port: 65_536 } }, + { route_probe: { version: 1, port: 1.5 } }, + { route_probe: { version: 1, port: "synthetic-private-value" } }, + { route_probe: { version: 1, port: 3000, extra: "private" } }, + { service: "web\nprivate" }, + ]) { + const detail = nativeFailureDetail({ ...reply, ...patch }); + expect(detail.message).toBeUndefined(); + expect(JSON.stringify(detail)).not.toContain("private"); + } +}); + bunTest.each([ { argument: "foreground-stopped", From 6f2f0782f0f942da9b583259e29922f496ea15c8 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:45:30 +0000 Subject: [PATCH 8/9] fix(native): correct failed-boot recovery and enforce boot ordering --- docs/guides/native-candidate.md | 18 ++- docs/reference/v5-model-coverage.md | 2 +- packages/runtime-core/README.md | 12 +- packages/runtime-core/src/provider/bridge.rs | 26 ++-- .../src/provider/bridge_startup_tests.rs | 17 +- .../runtime-core/src/provider/lifecycle.rs | 77 +++++++-- .../provider/lifecycle/boot_sequence_tests.rs | 146 ++++++++++++++++++ tests/native-project-start.test.ts | 4 +- tests/native-runtime-client.test.ts | 4 +- 9 files changed, 271 insertions(+), 35 deletions(-) create mode 100644 packages/runtime-core/src/provider/lifecycle/boot_sequence_tests.rs diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index 009e3a623..b2ef6fff4 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -705,8 +705,22 @@ The provider's guest-ready signal can arrive before its bridge threads bind. Nat rechecks the same provider identity around each observation; it never accepts an unsafe socket, changes its permissions or removes it to make startup pass. Existing running-pool audits do not wait. A socket timeout or identity refusal takes the verified failed-boot -stop path and names the recovery command: run the selected native executor's -`--candidate-root "$HACK_NATIVE_HOME" runtime down --recover --json`, then retry. This preserves +stop path and names the recovery command. Use the same selected executor and identical +`--candidate-root` argument as the failed start, followed by `runtime recover --json`. +If you explicitly selected an installed executor with `HACK_NATIVE_BINARY` and +`HACK_NATIVE_HOME`, use those same non-empty values: + +```sh +"$HACK_NATIVE_BINARY" --candidate-root "$HACK_NATIVE_HOME" runtime recover --json +``` + +Otherwise use the bundled executor path and the selected home from the failed +invocation; do not run this variable-based example with either variable unset. + +For a checkout-local executor, `--candidate-root` is the source checkout, not the native +home or `.hack-local` directory; from that checkout run `./hack-local runtime recover --json`. +Provider-verification failures during the wait carry this guidance too; recovery still +independently proves ownership before stopping anything. Then retry. This preserves retained disks and records an unclean stop; it does not claim the guest shut down cleanly. `hack down` may say the project was not started because no project mapping was published. diff --git a/docs/reference/v5-model-coverage.md b/docs/reference/v5-model-coverage.md index 41b0a0bbf..46fce6ca3 100644 --- a/docs/reference/v5-model-coverage.md +++ b/docs/reference/v5-model-coverage.md @@ -36,7 +36,7 @@ must be attached to the eventual exact integration revision. | Property and critical states | Model coverage | Implementation/regression boundary | Remaining qualification | | --- | --- | --- | --- | -| Guest control ready before exposed bridge sockets bind; identity changes or the startup deadline expires | No new TLA+ model; deterministic clock schedules both readiness orders against real Unix socket nodes | `provider/bridge.rs` waits only in fresh `finish_boot`, before adoption, and keeps running audits immediate. `bridge_startup_tests.rs` covers two slots, late creation and mode tightening, permanent absence/unsafe modes, type/UID/link refusals, provider loss before/after observation and late success rejection. No wait mutates sockets or grants cleanup authority; the existing verified-provider stop handles failure. `native-runtime-client.test.ts` checks the fixed recovery command reaches the frontend | Two sockets, finite scheduled events, injected provider verification and monotonic clock. Kernel process identity, provider scheduling, disk durability and successful recovery remain host boundaries. Require ten consecutive real-VM Mac Ctrl-C scenarios with pool stop/start; portable tests are not that evidence. | +| Guest control ready before exposed bridge sockets bind; identity changes or the startup deadline expires | No new TLA+ model; deterministic clock schedules both readiness orders against real Unix socket nodes | `provider/bridge.rs` waits only in fresh `finish_boot`, before adoption, and keeps running audits immediate. `bridge_startup_tests.rs` covers two slots, late creation and mode tightening, permanent absence/unsafe modes, type/UID/link refusals, provider loss before/after observation and late success rejection. No wait mutates sockets or grants cleanup authority; the existing verified-provider stop handles failure. `lifecycle/boot_sequence_tests.rs` drives the real waiter through the production fresh-boot sequence and rejects audit/adoption on timeout, unsafe sockets or provider loss. `native-runtime-client.test.ts` checks the fixed runtime recover command and identical-root hint reach the frontend | Two sockets, finite scheduled events, injected provider verification and monotonic clock. Kernel process identity, provider scheduling, disk durability and successful recovery remain host boundaries. Require ten consecutive real-VM Mac Ctrl-C scenarios with pool stop/start; portable tests are not that evidence. | | CI cancellation must not lose merged-change coverage; only exact-head full evidence may qualify a prerelease | No maintained GitHub scheduler model. Finite fixture interleavings cover predecessor success/failure/cancellation, two competing full/scoped runs and two rerun attempts | `ci.yml` serializes next pushes in one non-cancelling group (GitHub cancels only the replaced pending run), cancels stale PR runs, and requires a successful before run for selection. `selectPrereleaseCiRun` reads current attempts; `requireBranchChecks` binds suite and job check IDs. `ci-trusted-policy`, `prerelease-plan` and `release-prepare-gate` regressions reject cancelled predecessors, green subsets, stale checks, skipped hosts and changed attempts | API snapshots and GitHub group/event semantics are assumptions; races after the last read remain possible. Publication rechecks the frozen protected head and full evidence but does not transactionally lock GitHub. Hosted dispatch/cancellation checks qualify wiring; no scheduler liveness or formal refinement claim. | | Native log follower versus lifecycle mutation, frontend loss, stopped container and transport loss | No new state model: the follower observes without a mutation lease and grants no cleanup authority | `provider/graph/log_follow.rs` and its native counterpart pin Ready admission, container and start generation; `provider/engine/log_follow.rs` bounds framing and distinguishes real EOF from stopped-service idle cutoff. `lifecycle_log_follow_tests.rs` keeps the production transport on a fake Engine socket active while acquiring startup and operation leases, with a same-lock contention control and output while each lease is held. `native-log-follow-source.test.ts` guards the observer-only route; process tests cover cancellation, frontend loss, real output-pipe closure while idle, replacement-container restart (including retained adopted run/owner IDs) with one marker, NDJSON admission failures, truncated/partial output and slow idle-end delivery. `native-log-restart.test.ts` covers bounded re-admission, unchanged-container refusal and prompt down; `native-log-selection.test.ts` covers strict finite admission responses and owned-child cancellation/timeouts; advisory markers grant no authority | 4 KiB payload envelopes, 8 KiB wire lines, 64 Ki-character retained payload prefixes excluding Engine timestamps, 30-second restart wait, 1000-line new-container backlog. The portable lease test fakes identity/state checks; it does not exercise host process/disk auditing, graph admission or the VM relay. Real relay EOF on stop/down/restart, VM loss and concurrent lifecycle latency require Apple Silicon acceptance. No performance or whole-product parity claim. | | Foreground group interrupt; publisher exit before authenticated stop; dead owner's pending cleanup enrollment and bridge release | native-publisher-stop | [Process adapter](../../src/backends/native-project-process.ts) isolates Compose owners regardless of hooks; Rust `process::publisher_command` isolates publishers; `legacy_cleanup_enrollment` binds the old coordinator and inventories before a distinct recovery intent. [Group regression](../../tests/native-foreground-group-interrupt.test.ts) signals the whole group for fresh/restore owners; Rust real-publisher process, frontend-loss, exact child-reaping and enrollment tests check grouping, lifetime wakeups and stale selectors. | One graph, one publisher, one crash, one identity replacement and one audited successor boot; 168 states. Publisher exit observation is separate from lease-bound reaping/proof commitment; `NoUncommittedExit` rejects owner cleanup while proof is pending. A busy lease stutters without delaying stop control. SIGHUP requests retaining cleanup; a reaped publisher enters an observable publisher-down state until explicit stop. Native process tests and frontend framing/ps tests cover signal delivery and notices. Dependency release is a separate post-completion step; `NoStrandedReservation` rejects success with a held reservation. Real reservation-file regressions check retry, capacity reuse and refusal fences. Separate stopped-receipt and intent-completion commits are collapsed; TypeScript dispatch and Rust completion-digest regressions cover that interruption window. Publication-registry sync/rename is abstracted and covered by portable commit-interruption tests. Kernel identity, socket/inode proofs, guest observations, durable storage and existing next.13 home recovery remain implementation/Apple Silicon VM gates. No eventual recovery is asserted. | diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index e23749307..0365d473c 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -595,10 +595,16 @@ type, user, link count or unreadable identity refuses immediately. Only a comple socket set observed before the deadline succeeds. The wait never chmods, removes or connects to sockets. Running-pool audits remain immediate. The existing failed-boot path stops only the verified provider, records its absence and retains `failed-boot-stopped`, without -claiming a clean guest shutdown. Socket failures name the selected executor's -`runtime down --recover --json` on the same native home before retry; a project may not yet +claiming a clean guest shutdown. Socket failures and provider-verification failures during +the wait name `runtime recover --json`, using the same executor and identical +`--candidate-root` argument as the failed start. For an installed executor this argument is +the private `HACK_NATIVE_HOME`; for the checkout-local executor it is the source checkout +(the `hack-local` launcher supplies it). Do not substitute the `.hack-local` state directory. +A project may not yet have a mapping for ordinary `hack down`. The clock-driven real-socket regression reproduces -the ready-before-bind ordering without a VM, but cannot qualify actual provider scheduling. +the ready-before-bind ordering without a VM. A fresh-boot sequence regression runs the +actual waiter before audit/adoption and refuses timeout, unsafe sockets and provider loss +without adopting disks; host effects are injected and actual provider scheduling is not qualified. HACK-2175's live gate is ten consecutive Mac Ctrl-C scenarios including pool stop/start. The wait adds no signal handler: interruption retains the existing `booting` receipt and stop/recovery rules, so that live gate also qualifies the extended startup window. diff --git a/packages/runtime-core/src/provider/bridge.rs b/packages/runtime-core/src/provider/bridge.rs index 02ff13c52..11a083ca1 100644 --- a/packages/runtime-core/src/provider/bridge.rs +++ b/packages/runtime-core/src/provider/bridge.rs @@ -146,7 +146,7 @@ const BOOT_SOCKET_POLL: std::time::Duration = std::time::Duration::from_millis(5 fn startup_socket_timeout() -> CandidateError { CandidateError::new( "bridge_socket_timeout", - "Native bridge sockets did not become private and owned before the startup deadline. Run the selected native executor with --candidate-root set to this native home and runtime down --recover --json before retrying.", + "Native bridge sockets did not become private and owned before the startup deadline. Run the same selected executor with the identical --candidate-root argument used for the failed start, followed by runtime recover --json before retrying.", ) } @@ -159,14 +159,9 @@ pub(super) fn wait_for_sockets( directory: &std::path::Path, intent: BridgeIntent, verify_provider: impl FnMut() -> Result<(), CandidateError>, + clock: &impl super::process::Clock, ) -> Result<(), CandidateError> { - wait_for_sockets_with( - directory, - intent, - verify_provider, - &super::process::MonotonicClock::new(), - BOOT_SOCKET_WAIT, - ) + wait_for_sockets_with(directory, intent, verify_provider, clock, BOOT_SOCKET_WAIT) } fn wait_for_sockets_with( @@ -181,14 +176,14 @@ fn wait_for_sockets_with( if clock.now() >= deadline { return Err(startup_socket_timeout()); } - verify_provider()?; + verify_provider().map_err(startup_provider_failure)?; let ready = sockets_ready(directory, intent).map_err(|_| { CandidateError::new( "bridge_socket_identity", - "Bridge socket identity is unsafe; startup was refused. Run the selected native executor with --candidate-root set to this native home and runtime down --recover --json before retrying.", + "Bridge socket identity is unsafe; startup was refused. Run the same selected executor with the identical --candidate-root argument used for the failed start, followed by runtime recover --json before retrying.", ) })?; - verify_provider()?; + verify_provider().map_err(startup_provider_failure)?; let remaining = deadline.saturating_sub(clock.now()); if remaining.is_zero() { return Err(startup_socket_timeout()); @@ -200,6 +195,15 @@ fn wait_for_sockets_with( } } +/// Keep the failed identity boundary's code, but never expose its dynamic text. +/// Recovery still independently proves the recorded provider before any stop. +fn startup_provider_failure(failure: CandidateError) -> CandidateError { + CandidateError::new( + failure.code, + "The retained provider could not be verified while waiting for bridge sockets; startup was refused. Run the same selected executor with the identical --candidate-root argument used for the failed start, followed by runtime recover --json before retrying.", + ) +} + #[cfg(test)] #[path = "bridge_startup_tests.rs"] mod startup_tests; diff --git a/packages/runtime-core/src/provider/bridge_startup_tests.rs b/packages/runtime-core/src/provider/bridge_startup_tests.rs index 9e83aa6e7..3ef108abb 100644 --- a/packages/runtime-core/src/provider/bridge_startup_tests.rs +++ b/packages/runtime-core/src/provider/bridge_startup_tests.rs @@ -127,7 +127,12 @@ fn missing_or_never_private_sockets_expire_without_changing_them_and_name_recove .unwrap_err(); assert_eq!(failure.code, "bridge_socket_timeout"); assert!(failure.message_fixed); - assert!(failure.message.contains("runtime down --recover --json")); + assert!(failure.message.contains("runtime recover --json")); + assert!( + failure + .message + .contains("identical --candidate-root argument") + ); assert_eq!(clock.now(), BUDGET); if present { assert_eq!( @@ -165,7 +170,7 @@ fn unsafe_type_alias_and_link_count_refuse_even_when_an_earlier_slot_is_missing( .unwrap_err(); assert_eq!(failure.code, "bridge_socket_identity"); assert!(failure.message_fixed); - assert!(failure.message.contains("runtime down --recover --json")); + assert!(failure.message.contains("runtime recover --json")); let after = fs::symlink_metadata(fixture.path(1)).unwrap(); assert_eq!( (before.dev(), before.ino(), before.mode(), before.nlink()), @@ -215,6 +220,14 @@ fn provider_loss_before_or_after_observation_is_not_readiness_or_retry_authority ) .unwrap_err(); assert_eq!(failure.code, "process_identity_mismatch"); + assert!(failure.message_fixed); + assert!(failure.message.contains("runtime recover --json")); + assert!( + failure + .message + .contains("identical --candidate-root argument") + ); + assert!(!failure.message.contains("Provider changed.")); assert_eq!(checks.get(), fail_on); assert_eq!(clock.now(), Duration::ZERO); } diff --git a/packages/runtime-core/src/provider/lifecycle.rs b/packages/runtime-core/src/provider/lifecycle.rs index 4fc8aa14a..88bfdf906 100644 --- a/packages/runtime-core/src/provider/lifecycle.rs +++ b/packages/runtime-core/src/provider/lifecycle.rs @@ -1604,7 +1604,7 @@ fn start_pool( if cleanup.is_ok() { phase(candidate, &mut owner, "failed-boot-stopped")?; } - cleanup?; + cleanup.map_err(failed_boot_stop_error)?; } result } @@ -1694,28 +1694,81 @@ fn verify_disk_allocation( Ok(()) } +#[cfg(test)] +#[path = "lifecycle/boot_sequence_tests.rs"] +mod boot_sequence_tests; + +fn failed_boot_stop_error(failure: CandidateError) -> CandidateError { + CandidateError::new( + failure.code, + "Failed-boot provider cleanup could not be confirmed. Run the same selected executor with the identical --candidate-root argument used for the failed start, followed by runtime recover --json before retrying.", + ) +} + fn finish_boot( candidate: &Candidate, owner: &mut Owner, previous_boot: Option<&str>, lock: &state::Lock, ) -> Result { - // Before any disk is adopted: a refusal here takes the caller's failed-boot path. - verify_templates( - candidate, + finish_boot_with( owner, - lock, - super::disk_template::Stage::AfterFirstStart, - )?; + |owner| { + verify_templates( + candidate, + owner, + lock, + super::disk_template::Stage::AfterFirstStart, + )?; + owner + .application_bridge + .map(|bridge| { + owner + .real_data_dir(candidate) + .map(|directory| (directory, bridge)) + }) + .transpose() + }, + |owner| verify_live(candidate, owner), + |owner| audit_boot(candidate, owner), + |owner| adopt_boot(candidate, owner, previous_boot, lock), + &process::MonotonicClock::new(), + ) +} + +/// Fresh boot's ordering boundary. Only effectful host/guest operations are +/// injected: the real complete-set socket wait always precedes audit/adoption. +fn finish_boot_with( + owner: &mut S, + prepare: impl FnOnce( + &S, + ) -> Result< + Option<(std::path::PathBuf, super::bridge::BridgeIntent)>, + CandidateError, + >, + mut verify_provider: impl FnMut(&S) -> Result<(), CandidateError>, + audit: impl FnOnce(&S) -> Result<(), CandidateError>, + adopt: impl FnOnce(&mut S) -> Result, + clock: &impl process::Clock, +) -> Result { + // Before any disk is adopted: a refusal here takes the caller's failed-boot path. + let bridge = prepare(owner)?; // The provider's agent-ready signal can precede expose-thread socket binds. // Wait before adoption, without weakening the immediate audit used by // running-pool commands. Failure keeps the existing verified-stop path. - if let Some(bridge) = owner.application_bridge { - super::bridge::wait_for_sockets(&owner.real_data_dir(candidate)?, bridge, || { - verify_live(candidate, owner) - })?; + if let Some((directory, bridge)) = bridge { + super::bridge::wait_for_sockets(&directory, bridge, || verify_provider(owner), clock)?; } - audit_boot(candidate, owner)?; + audit(owner)?; + adopt(owner) +} + +fn adopt_boot( + candidate: &Candidate, + owner: &mut Owner, + previous_boot: Option<&str>, + lock: &state::Lock, +) -> Result { let dir = owner.real_data_dir(candidate)?; let storage = identity::disk(&dir.join("storage.raw"))?; let overlay = identity::disk(&dir.join("overlay.raw"))?; diff --git a/packages/runtime-core/src/provider/lifecycle/boot_sequence_tests.rs b/packages/runtime-core/src/provider/lifecycle/boot_sequence_tests.rs new file mode 100644 index 000000000..fb5ab1c79 --- /dev/null +++ b/packages/runtime-core/src/provider/lifecycle/boot_sequence_tests.rs @@ -0,0 +1,146 @@ +//! The production fresh-boot sequence drives the real socket waiter before +//! audit/adoption. Host template/process/disk effects are injected, not qualified. +use super::*; +use std::{ + cell::{Cell, RefCell}, + os::unix::{fs::PermissionsExt, net::UnixListener}, +}; + +struct Clock<'a> { + elapsed: Cell, + on_sleep: RefCell>, +} +impl process::Clock for Clock<'_> { + fn now(&self) -> Duration { + self.elapsed.get() + } + fn sleep(&self, duration: Duration) { + self.elapsed.set(self.elapsed.get() + duration); + (self.on_sleep.borrow_mut())(); + } +} + +#[test] +fn fresh_boot_waits_before_audit_and_never_adopts_failed_socket_readiness() { + for mode in [ + "delayed", + "missing", + "unsafe", + "provider-before", + "provider-after", + ] { + let directory = crate::unique_test_root(&std::env::temp_dir(), "boot-bridge"); + let path = directory.join("bridge-00.sock"); + if mode == "unsafe" { + fs::write(&path, b"foreign").unwrap(); + } + let listener = RefCell::new(None); + let clock = Clock { + elapsed: Cell::new(Duration::ZERO), + on_sleep: RefCell::new(Box::new(|| { + if mode == "delayed" && listener.borrow().is_none() { + *listener.borrow_mut() = Some(UnixListener::bind(&path).unwrap()); + fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); + } + })), + }; + let events = RefCell::new(Vec::new()); + let checks = Cell::new(0); + let mut adopted = false; + let result = finish_boot_with( + &mut adopted, + |_| { + events.borrow_mut().push("templates"); + Ok(Some(( + directory.clone(), + super::super::bridge::BridgeIntent::new(1)?, + ))) + }, + |_| { + checks.set(checks.get() + 1); + let lose = (mode == "provider-before" && checks.get() == 1) + || (mode == "provider-after" && checks.get() == 2); + if lose { + Err(CandidateError::new( + "process_identity_mismatch", + "private detail", + )) + } else { + Ok(()) + } + }, + |_| { + // Deleting the production wait now fails this assertion: the + // scheduled bind has not occurred when audit runs immediately. + super::super::bridge::verify_sockets( + &directory, + super::super::bridge::BridgeIntent::new(1)?, + )?; + events.borrow_mut().push("audit"); + Ok(()) + }, + |adopted| { + *adopted = true; + events.borrow_mut().push("adopt"); + Ok(()) + }, + &clock, + ); + if mode == "delayed" { + result.unwrap(); + assert!(adopted); + assert_eq!(clock.elapsed.get(), Duration::from_millis(50)); + assert_eq!(*events.borrow(), ["templates", "audit", "adopt"]); + } else { + let failure = result.unwrap_err(); + assert!(!adopted); + assert_eq!(*events.borrow(), ["templates"]); + let code = match mode { + "missing" => "bridge_socket_timeout", + "unsafe" => "bridge_socket_identity", + _ => "process_identity_mismatch", + }; + assert_eq!(failure.code, code); + assert!(failure.message_fixed); + assert!(failure.message.contains("runtime recover --json")); + } + drop(clock); + drop(listener.into_inner()); + fs::remove_dir_all(directory).unwrap(); + } +} + +#[test] +fn template_refusal_cannot_reach_socket_observation_audit_or_adoption() { + let clock = Clock { + elapsed: Cell::new(Duration::ZERO), + on_sleep: RefCell::new(Box::new(|| panic!("no wait"))), + }; + let failure = finish_boot_with( + &mut (), + |_| Err(CandidateError::new("disk_template_untrusted", "fixture")), + |_| panic!("no socket observation"), + |_| panic!("no audit"), + |_| -> Result<(), CandidateError> { panic!("no adoption") }, + &clock, + ) + .unwrap_err(); + assert_eq!(failure.code, "disk_template_untrusted"); +} + +#[test] +fn failed_boot_stop_preserves_its_code_and_the_recovery_command() { + let failure = failed_boot_stop_error(CandidateError::new( + "process_identity_unavailable", + String::from("private transport detail"), + )); + assert_eq!(failure.code, "process_identity_unavailable"); + assert!(failure.message_fixed); + assert!(failure.message.contains("runtime recover --json")); + assert!( + failure + .message + .contains("identical --candidate-root argument") + ); + assert!(!failure.message.contains("private transport detail")); +} diff --git a/tests/native-project-start.test.ts b/tests/native-project-start.test.ts index 3b3a3abb2..d04ffab54 100644 --- a/tests/native-project-start.test.ts +++ b/tests/native-project-start.test.ts @@ -426,7 +426,7 @@ test("foreground saves after readiness and retains mapping after confirmed stop" test("a pool bridge boot failure reaches foreground up with recovery guidance before any project mapping", async () => { const { opts, events } = await fixture(false); const message = - "Native bridge sockets did not become private and owned before the startup deadline. Run the selected native executor with --candidate-root set to this native home and runtime down --recover --json before retrying."; + "Native bridge sockets did not become private and owned before the startup deadline. Run the same selected executor with the identical --candidate-root argument used for the failed start, followed by runtime recover --json before retrying."; await writeFile( opts.runtime.binary, `#!/usr/bin/env bun @@ -446,7 +446,7 @@ process.exit(2); (error: unknown) => error ); expect(failure).toBeInstanceOf(ClosedCliError); - expect(String(failure)).toContain("runtime down --recover --json"); + expect(String(failure)).toContain("runtime recover --json"); expect(String(failure)).toContain("--candidate-root"); expect(String(failure)).not.toContain("synthetic-secret"); expect(nativeRuntimeFailure(failure)?.code).toBe("bridge_socket_timeout"); diff --git a/tests/native-runtime-client.test.ts b/tests/native-runtime-client.test.ts index 3576d0a56..8184300a6 100644 --- a/tests/native-runtime-client.test.ts +++ b/tests/native-runtime-client.test.ts @@ -149,7 +149,7 @@ async function fixture() { if (process.argv.includes("hang")) await Bun.sleep(60_000); if (process.argv.includes("old-provider")) { console.error(JSON.stringify({code:"provider_pin_changed",message:"synthetic-secret-diagnostic"})); process.exit(2); } if (process.argv.includes("fixed-message")) { console.error(JSON.stringify({code:"graph_image",message:"Pinned image architecture or implicit volumes are incompatible.",message_fixed:true,service:"web"})); process.exit(2); } -if (process.argv.includes("bridge-socket-failure")) { console.error(JSON.stringify({code:"bridge_socket_timeout",message:"Native bridge sockets did not become private and owned before the startup deadline. Run the selected native executor with --candidate-root set to this native home and runtime down --recover --json before retrying.",message_fixed:true})); process.exit(2); } +if (process.argv.includes("bridge-socket-failure")) { console.error(JSON.stringify({code:"bridge_socket_timeout",message:"Native bridge sockets did not become private and owned before the startup deadline. Run the same selected executor with the identical --candidate-root argument used for the failed start, followed by runtime recover --json before retrying.",message_fixed:true})); process.exit(2); } if (process.argv.includes("unsafe-fixed")) { console.error(JSON.stringify({code:"graph_image",message:"synthetic-secret-diagnostic\\nsecond line",message_fixed:true,service:"../synthetic-secret-diagnostic"})); process.exit(2); } if (process.argv.includes("plan-change")) { console.error(JSON.stringify({code:"graph_plan_reconcile",message:"synthetic-secret-diagnostic",plan_change:{version:1,service:"deps",field:"dependency_cache_volume"}})); process.exit(23); } if (process.argv.includes("plan-change-unsafe")) { console.error(JSON.stringify({code:"graph_hostname_change",message:"synthetic-secret-diagnostic",plan_change:{version:1,service:"synthetic secret diagnostic",field:"hostnames"}})); process.exit(23); } @@ -1010,7 +1010,7 @@ test("a failed bridge boot names its one recovery command through the runtime cl }).catch((caught: unknown) => caught); expect(error).toBeInstanceOf(NativeRuntimeRequestError); expect(error).toHaveProperty("nativeCode", "bridge_socket_timeout"); - expect(String(error)).toContain("runtime down --recover --json"); + expect(String(error)).toContain("runtime recover --json"); expect(String(error)).not.toContain("nothing was started"); }); From 5ab03504c235fed60c1fb3bc04a43a1a48cc318d Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:48:11 +0000 Subject: [PATCH 9/9] test(native): require route readiness with the production grace --- .../src/project/route_probe_clock_tests.rs | 102 ++++++++---------- 1 file changed, 47 insertions(+), 55 deletions(-) diff --git a/packages/runtime-core/src/project/route_probe_clock_tests.rs b/packages/runtime-core/src/project/route_probe_clock_tests.rs index a6d1a5c59..6e8c698ee 100644 --- a/packages/runtime-core/src/project/route_probe_clock_tests.rs +++ b/packages/runtime-core/src/project/route_probe_clock_tests.rs @@ -56,59 +56,51 @@ fn a_route_first_answering_at_120_seconds_starts_under_the_graph_budget() { }, )])) .unwrap(); - // The old grace is a negative control: the same delayed success must fail - // early with it. The reviewed default must reach graph readiness at 120 s. - for grace in [60_000, HttpProbe::inferred_route(3000).start_period_ms] { - let mut config = HttpProbe::inferred_route(3000); - config.start_period_ms = grace; - let output = Command::new(concat!(env!("OUT_DIR"), "/http-probe-clock-host")) - .args([ - grace.to_string(), - config.interval_ms.to_string(), - config.timeout_ms.to_string(), - config.retries.to_string(), - "120000".into(), - ]) - .output() - .unwrap(); - assert!(output.status.success()); - assert!(output.stderr.is_empty()); - let samples = String::from_utf8(output.stdout) - .unwrap() - .lines() - .map(|line| { - let fields: Vec = line - .split(' ') - .map(|field| field.parse().unwrap()) - .collect(); - (fields[0], fields[1], fields[2]) - }) - .collect(); - let clock = Cell::new(Duration::ZERO); - let base = Instant::now(); - let mut driver = Probed { - clock: &clock, - config, - samples, - started: false, - ready: false, - }; - let result = run_with_clock( - &graph, - &mut driver, - Duration::from_secs(300), - |delay| clock.set(clock.get() + delay), - || base + clock.get(), - ); - assert!(driver.started); - if grace == 60_000 { - assert_eq!(result.unwrap_err().code, "graph_service_failed"); - assert!(!driver.ready); - assert!(clock.get() < Duration::from_secs(120)); - } else { - result.unwrap(); - assert!(driver.ready); - assert_eq!(clock.get(), Duration::from_secs(120)); - } - } + // Use the production default unchanged, and always require readiness. + // Restoring its grace to 60 s must fail this test, not select a passing + // branch that expects the service to become unhealthy. + let config = HttpProbe::inferred_route(3000); + let output = Command::new(concat!(env!("OUT_DIR"), "/http-probe-clock-host")) + .args([ + config.start_period_ms.to_string(), + config.interval_ms.to_string(), + config.timeout_ms.to_string(), + config.retries.to_string(), + "120000".into(), + ]) + .output() + .unwrap(); + assert!(output.status.success()); + assert!(output.stderr.is_empty()); + let samples = String::from_utf8(output.stdout) + .unwrap() + .lines() + .map(|line| { + let fields: Vec = line + .split(' ') + .map(|field| field.parse().unwrap()) + .collect(); + (fields[0], fields[1], fields[2]) + }) + .collect(); + let clock = Cell::new(Duration::ZERO); + let base = Instant::now(); + let mut driver = Probed { + clock: &clock, + config, + samples, + started: false, + ready: false, + }; + let result = run_with_clock( + &graph, + &mut driver, + Duration::from_secs(300), + |delay| clock.set(clock.get() + delay), + || base + clock.get(), + ); + assert!(driver.started); + result.expect("the production inferred probe must allow readiness at 120 s"); + assert!(driver.ready); + assert_eq!(clock.get(), Duration::from_secs(120)); }