From 5f69370c01a96263654aa4d42a610adfdac914f2 Mon Sep 17 00:00:00 2001 From: Shiju Date: Tue, 20 Oct 2026 11:25:09 +0530 Subject: [PATCH 1/2] fix(gateway): report isolation backend mismatches in sandbox status When a sandbox's pinned isolation backend no longer matched the gateway's selection, the automatic restart scan only logged a warning every 500 ms and left the sandbox in Starting with a stale restart reason. A sandbox that never started later failed with ImagePreparationTimedOut. The restart scan now moves the sandbox to Error with IsolationBackendMismatch, writing under the lock it holds with a compare-and-swap on the version it read. A preparation timeout reports the same reason when the backend does not match; an admission repair timeout keeps ProvisioningTimedOut. Startup and lifecycle recovery still skip mismatched sandboxes, because they run on every replica and a skipped sandbox may be running on the backend it was created with. Closes #4737 Signed-off-by: Shiju --- crates/openshell-server/src/compute/mod.rs | 180 +++++++++++++++++- .../src/compute/provisioning_deadline.rs | 30 ++- docs/how-it-works/sandboxes/overview.mdx | 2 + 3 files changed, 200 insertions(+), 12 deletions(-) diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index a100aa11d1..95e33573e5 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -4420,8 +4420,42 @@ impl ComputeRuntime { } let sandbox_name = current.object_name().to_string(); - self.validate_isolation_backend_binding(¤t) - .map_err(|error| error.to_string())?; + if let Err(error) = self.validate_isolation_backend_binding(¤t) { + // Only an explicit stop and start can relaunch the sandbox once the + // gateway selection matches again, so leave the restart scan. + let message = error.message().to_string(); + match self + .store + .update_message_cas::( + sandbox_id, + sandbox_resource_version(¤t), + |sandbox| { + let status = sandbox.status.get_or_insert_with(Default::default); + set_next_restart_at_ms(status, 0); + sandbox.set_phase(SandboxPhase::Error as i32); + upsert_ready_condition( + &mut sandbox.status, + SandboxCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: "IsolationBackendMismatch".to_string(), + message: message.clone(), + transition_time: None, + }, + ); + }, + ) + .await + { + Ok(updated) => { + self.sandbox_index.update_from_sandbox(&updated); + self.sandbox_watch_bus.notify(sandbox_id); + } + Err(crate::persistence::PersistenceError::Conflict { .. }) => return Ok(()), + Err(err) => return Err(err.to_string()), + } + return Err(error.to_string()); + } let already_claimed = current .status .as_ref() @@ -10526,6 +10560,64 @@ mod tests { assert!(next_restart_at_ms(&stored.status.unwrap()) > openshell_core::time::now_ms()); } + #[tokio::test] + async fn restart_scan_reports_isolation_backend_mismatch_once() { + let driver = ControlledDriver::new(); + let runtime = test_runtime(driver.clone()).await; + let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Starting); + sandbox + .metadata + .as_mut() + .unwrap() + .annotations + .insert(ISOLATION_BACKEND_ANNOTATION.into(), "other-backend".into()); + sandbox.status = Some(SandboxStatus { + phase: SandboxPhase::Starting as i32, + exit_code: Some(9), + restart_count: 1, + next_restart_time: restart_timestamp(openshell_core::time::now_ms() - 1), + ..Default::default() + }); + runtime.store.put_message(&sandbox).await.unwrap(); + + runtime.restart_due_sandboxes().await.unwrap(); + let failed = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + runtime.restart_due_sandboxes().await.unwrap(); + + assert_eq!(driver.stop_calls(), 0); + assert_eq!(driver.start_calls(), 0); + assert_eq!(failed.phase(), SandboxPhase::Error as i32); + assert!( + failed + .status + .as_ref() + .unwrap() + .conditions + .iter() + .any(|condition| { + condition.r#type == "Ready" + && condition.reason == "IsolationBackendMismatch" + && condition.message.contains("restore the gateway selection") + }) + ); + let rescanned = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + assert_eq!( + sandbox_resource_version(&rescanned), + sandbox_resource_version(&failed), + "the next scan must leave a mismatched sandbox alone" + ); + } + #[tokio::test] async fn concurrent_stop_transition_prevents_replacement_start() { let driver = ControlledDriver::new(); @@ -17277,6 +17369,17 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" driver.calls().is_empty(), "another replica must not launch a differently bound resource" ); + assert_eq!( + runtime + .store + .get_message::("external-start") + .await + .unwrap() + .unwrap() + .phase(), + SandboxPhase::Starting as i32, + "another replica must not fail a launch it cannot run" + ); } let mut capabilities = driver .get_capabilities(Request::new(GetCapabilitiesRequest::default())) @@ -19436,6 +19539,79 @@ endpoint = { kind = "unix", socket_path = "/run/agent-substrate/isolation.sock" assert_eq!(driver.stop_calls(), 1); } + #[tokio::test] + async fn preparation_timeout_reports_isolation_backend_mismatch() { + let driver = ControlledDriver::new(); + let runtime = test_runtime(driver.clone()).await; + let mut sandbox = sandbox_record("sb-prepare", "prepare", SandboxPhase::Starting); + sandbox + .metadata + .as_mut() + .unwrap() + .annotations + .insert(ISOLATION_BACKEND_ANNOTATION.into(), "other-backend".into()); + sandbox.status.as_mut().unwrap().provisioning = + Some(provisioning_deadline::new_preparation_record(0, 1800)); + runtime.store.put_message(&sandbox).await.unwrap(); + let sandbox = runtime + .store + .get_message::("sb-prepare") + .await + .unwrap() + .unwrap(); + let gate = runtime.lifecycle_gates.lock_for("sb-prepare").await; + let _sandbox_guard = runtime.lock_sandbox_for_lifecycle(&gate).await; + let expired = runtime + .claim_provisioning_timeout(&sandbox, 1_800_000) + .await + .unwrap() + .unwrap(); + assert_eq!(expired.phase(), i32::from(SandboxPhase::Error)); + assert!( + expired + .status + .as_ref() + .unwrap() + .conditions + .iter() + .any(|condition| { + condition.reason == "IsolationBackendMismatch" + && condition.message.contains("restore the gateway selection") + }) + ); + + // A registered supervisor keeps the admission diagnostic. + let mut admitted = sandbox_record("sb-admission", "admission", SandboxPhase::Starting); + admitted + .metadata + .as_mut() + .unwrap() + .annotations + .insert(ISOLATION_BACKEND_ANNOTATION.into(), "other-backend".into()); + admitted.status.as_mut().unwrap().provisioning = Some(provisioning_deadline::new_record(0)); + runtime.store.put_message(&admitted).await.unwrap(); + let admitted = runtime + .store + .get_message::("sb-admission") + .await + .unwrap() + .unwrap(); + let expired = runtime + .claim_provisioning_timeout(&admitted, 300_000) + .await + .unwrap() + .unwrap(); + assert!( + expired + .status + .as_ref() + .unwrap() + .conditions + .iter() + .any(|condition| condition.reason == "ProvisioningTimedOut") + ); + } + #[tokio::test] async fn preparation_timeout_retains_reason_and_uses_existing_cleanup() { let driver = ControlledDriver::new(); diff --git a/crates/openshell-server/src/compute/provisioning_deadline.rs b/crates/openshell-server/src/compute/provisioning_deadline.rs index 7b7608d38d..9385e1c836 100644 --- a/crates/openshell-server/src/compute/provisioning_deadline.rs +++ b/crates/openshell-server/src/compute/provisioning_deadline.rs @@ -552,6 +552,12 @@ impl super::ComputeRuntime { }; let mut deadline = ProvisioningDeadline::from_record(record)?; let preparation_expired = deadline.is_preparing(); + // A supervisor that never registered may be waiting on a launch bound + // to another isolation backend; report that instead of a timeout. + let backend_mismatch = preparation_expired + .then(|| self.validate_isolation_backend_binding(current).err()) + .flatten() + .map(|error| error.message().to_string()); if !deadline.expire(&record.attempt_id, &record.configuration_change_id, now_ms) { return Ok(None); } @@ -573,22 +579,26 @@ impl super::ComputeRuntime { .configuration_admission .as_ref() .map_or("", |admission| admission.error.as_str()); - let message = if preparation_expired { - "Image preparation or initial supervisor startup exceeded its absolute deadline".to_string() - } else if diagnostic.is_empty() { - "Provisioning repair window expired after 300 seconds".to_string() - } else { - format!( - "Provisioning repair window expired after 300 seconds: {diagnostic}" - ) - }; + let message = backend_mismatch.clone().unwrap_or_else(|| { + if preparation_expired { + "Image preparation or initial supervisor startup exceeded its absolute deadline".to_string() + } else if diagnostic.is_empty() { + "Provisioning repair window expired after 300 seconds".to_string() + } else { + format!( + "Provisioning repair window expired after 300 seconds: {diagnostic}" + ) + } + }); status .conditions .retain(|condition| condition.r#type != "Ready"); status.conditions.push(SandboxCondition { r#type: "Ready".into(), status: "False".into(), - reason: if preparation_expired { + reason: if backend_mismatch.is_some() { + "IsolationBackendMismatch" + } else if preparation_expired { "ImagePreparationTimedOut" } else { "ProvisioningTimedOut" diff --git a/docs/how-it-works/sandboxes/overview.mdx b/docs/how-it-works/sandboxes/overview.mdx index 5a84718039..cc685468bf 100644 --- a/docs/how-it-works/sandboxes/overview.mdx +++ b/docs/how-it-works/sandboxes/overview.mdx @@ -1045,6 +1045,8 @@ The first authenticated configuration report from the current supervisor ends pr Preparation expiry sets the sandbox to `Error` with reason `ImagePreparationTimedOut`; admission repair expiry uses `ProvisioningTimedOut`. The gateway stops its workload and supervisor compute, retaining the sandbox record, diagnostic, and restartable storage. Cleanup can remain pending if the backend is unavailable; the gateway retries it. Inspect `provisioning` in JSON output for the active `phase`, `deadline`, original `preparation_deadline`, `admission_start_time`, and cleanup timestamps. TUI NOTES identifies preparation timeout and distinguishes pending cleanup from reclaimed compute. +If the sandbox was created with an isolation backend that no longer matches the gateway's selection, preparation expiry uses reason `IsolationBackendMismatch` instead. An automatic restart that finds the mismatch sets `Error` with the same reason without waiting for a deadline. Restore the gateway selection or create a new sandbox. After a mismatch timeout, wait for cleanup and start the sandbox as described below. After an automatic restart fails this way, stop the sandbox, then start it. + The gateway keeps each submitted create or start operation running after its caller disconnects, its deadline expires, or monitoring fails. This includes recovery and the stop/start sequence for automatic restart. While `driver_operation_pending` is true, explicit start, stop, and deletion are blocked; automatic restart keeps its schedule and waits. Timeout cleanup can try to stop partial compute, but it cannot declare completion until the original driver call returns and a subsequent stop succeeds. This prevents a stop that observed missing compute from allowing a retry just before the original create finishes. Uploaded rootfs archives remain available to the owned operation. An actual driver response clears the pending flag, including ordinary rejection errors. The gateway retains `driver_operation_id` so a delayed response handler cannot change newer recovery work on the same provisioning attempt. Both fields appear in the `provisioning` JSON object. Backend behavior after a transport error still follows the driver's existing contract. If the owning gateway crashes before recording the response, the flag and any protected upload can remain indefinitely; restart and age alone cannot establish that the operation finished. There is currently no API to resolve that lost ownership, so automatic cleanup completion, retry, and deletion remain blocked for that record. From c609dc22a29f579b0bc74470e0955373e5e8e755 Mon Sep 17 00:00:00 2001 From: Shiju Date: Wed, 21 Oct 2026 11:56:26 +0530 Subject: [PATCH 2/2] docs: drop backend mismatch paragraph from sandbox overview --- docs/how-it-works/sandboxes/overview.mdx | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/how-it-works/sandboxes/overview.mdx b/docs/how-it-works/sandboxes/overview.mdx index cc685468bf..5a84718039 100644 --- a/docs/how-it-works/sandboxes/overview.mdx +++ b/docs/how-it-works/sandboxes/overview.mdx @@ -1045,8 +1045,6 @@ The first authenticated configuration report from the current supervisor ends pr Preparation expiry sets the sandbox to `Error` with reason `ImagePreparationTimedOut`; admission repair expiry uses `ProvisioningTimedOut`. The gateway stops its workload and supervisor compute, retaining the sandbox record, diagnostic, and restartable storage. Cleanup can remain pending if the backend is unavailable; the gateway retries it. Inspect `provisioning` in JSON output for the active `phase`, `deadline`, original `preparation_deadline`, `admission_start_time`, and cleanup timestamps. TUI NOTES identifies preparation timeout and distinguishes pending cleanup from reclaimed compute. -If the sandbox was created with an isolation backend that no longer matches the gateway's selection, preparation expiry uses reason `IsolationBackendMismatch` instead. An automatic restart that finds the mismatch sets `Error` with the same reason without waiting for a deadline. Restore the gateway selection or create a new sandbox. After a mismatch timeout, wait for cleanup and start the sandbox as described below. After an automatic restart fails this way, stop the sandbox, then start it. - The gateway keeps each submitted create or start operation running after its caller disconnects, its deadline expires, or monitoring fails. This includes recovery and the stop/start sequence for automatic restart. While `driver_operation_pending` is true, explicit start, stop, and deletion are blocked; automatic restart keeps its schedule and waits. Timeout cleanup can try to stop partial compute, but it cannot declare completion until the original driver call returns and a subsequent stop succeeds. This prevents a stop that observed missing compute from allowing a retry just before the original create finishes. Uploaded rootfs archives remain available to the owned operation. An actual driver response clears the pending flag, including ordinary rejection errors. The gateway retains `driver_operation_id` so a delayed response handler cannot change newer recovery work on the same provisioning attempt. Both fields appear in the `provisioning` JSON object. Backend behavior after a transport error still follows the driver's existing contract. If the owning gateway crashes before recording the response, the flag and any protected upload can remain indefinitely; restart and age alone cannot establish that the operation finished. There is currently no API to resolve that lost ownership, so automatic cleanup completion, retry, and deletion remain blocked for that record.