diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 17b57a29c1..bfc59fb4c9 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -77,7 +77,7 @@ use std::sync::Arc; use std::time::{Duration, SystemTime}; use tokio::sync::{OnceCell, mpsc}; use tokio_stream::wrappers::ReceiverStream; -use tracing::{debug, info, warn}; +use tracing::{debug, error, info, warn}; pub type WatchStream = Pin> + Send>>; @@ -4395,6 +4395,15 @@ impl KubernetesComputeDriver { let Some(name) = object.metadata.name.as_deref() else { return; }; + // A fail-closed suspension is permanent and otherwise looks like a + // deliberate stop, so it has to say so. + if cause == SandboxRuntimeSuspensionCause::FailClosed { + error!( + sandbox = name, + operation = sandbox_runtime_bootstrap_operation(object).unwrap_or(""), + "suspending sandbox-runtime fail-closed; the sandbox stays stopped until someone acts on it" + ); + } let namespace = object .metadata .namespace @@ -4443,12 +4452,13 @@ impl KubernetesComputeDriver { if sandbox_runtime_bootstrap_operation(object) != Some("create") { warn!( sandbox = object.metadata.name.as_deref().unwrap_or(""), - "suspending stale sandbox-runtime restart bootstrap" + operation = sandbox_runtime_bootstrap_operation(object).unwrap_or(""), + "suspending stale sandbox-runtime bootstrap" ); self.suspend_sandbox_runtime_after_dependency_failure( lookup_api, object, - SandboxRuntimeSuspensionCause::FailClosed, + stale_bootstrap_suspension_cause(object), ) .await; return; @@ -4950,6 +4960,21 @@ fn sandbox_runtime_bootstrap_phase_patch( }) } +/// Why a stale bootstrap that is not a create gets suspended. A restart that +/// outlasts the grace is usually waiting for somewhere to run, as during a node +/// rotation. The sandbox was meant to be running, so it is a lost runtime: the +/// workload is suspended all the same, and the gateway starts a fresh generation +/// under its restart backoff. Failing closed here would leave it Stopped for +/// good, with nothing to say it did not stop on purpose. Anything else, a stop +/// above all, stays stopped. +fn stale_bootstrap_suspension_cause(object: &DynamicObject) -> SandboxRuntimeSuspensionCause { + if sandbox_runtime_bootstrap_operation(object) == Some("restart") { + SandboxRuntimeSuspensionCause::RuntimeLost + } else { + SandboxRuntimeSuspensionCause::FailClosed + } +} + fn sandbox_runtime_bootstrap_is_stale( object: &DynamicObject, now: SystemTime, @@ -8845,6 +8870,51 @@ mod tests { assert!(!sandbox_runtime_bootstrap_in_progress(&object)); } + #[test] + fn stale_restart_bootstrap_is_a_lost_runtime_and_a_stale_stop_stays_stopped() { + let with_operation = |operation: Option<&str>| { + let mut annotations = serde_json::Map::new(); + annotations.insert( + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING.to_string(), + serde_json::json!("true"), + ); + if let Some(operation) = operation { + annotations.insert( + ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_OPERATION.to_string(), + serde_json::json!(operation), + ); + } + serde_json::from_value::(serde_json::json!({ + "metadata": {"annotations": annotations} + })) + .unwrap() + }; + assert_eq!( + stale_bootstrap_suspension_cause(&with_operation(Some("restart"))), + SandboxRuntimeSuspensionCause::RuntimeLost + ); + for operation in [Some("stop"), None] { + assert_eq!( + stale_bootstrap_suspension_cause(&with_operation(operation)), + SandboxRuntimeSuspensionCause::FailClosed, + "operation {operation:?}" + ); + } + + // The cause is what the gateway reads to replace the runtime, and the + // workload is suspended either way. + let patch = sandbox_runtime_dependency_suspension_patch( + SANDBOX_VERSION_V1BETA1, + "7", + stale_bootstrap_suspension_cause(&with_operation(Some("restart"))), + ); + assert_eq!(patch["spec"]["operatingMode"], "Suspended"); + assert_eq!( + patch["metadata"]["annotations"][ANNOTATION_SANDBOX_RUNTIME_SUSPENSION_CAUSE], + SANDBOX_RUNTIME_SUSPENSION_CAUSE_RUNTIME_LOST + ); + } + #[test] fn sandbox_owner_identity_requires_matching_uid_and_sandbox_id() { let owner = sandbox_owner_for_test("sandbox-a", "sandbox-uid-a");