diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index d0309a2efd..4e5820df46 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -44,6 +44,7 @@ use oci_client::manifest::{ use oci_client::secrets::RegistryAuth; use oci_client::{Reference, RegistryOperation}; use openshell_core::UpstreamProxyConfig; +use openshell_core::config::DEFAULT_STOP_TIMEOUT_SECS; use openshell_core::driver_mounts; use openshell_core::gpu::{ driver_gpu_requirements, effective_driver_gpu_count, validate_specific_gpu_device_request, @@ -90,9 +91,11 @@ use std::os::fd::AsRawFd as _; use std::os::fd::OwnedFd; #[cfg(unix)] use std::os::unix::fs::PermissionsExt; +#[cfg(unix)] +use std::os::unix::process::ExitStatusExt as _; use std::path::{Component, Path, PathBuf}; use std::pin::Pin; -use std::process::Stdio; +use std::process::{ExitStatus, Stdio}; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::Duration; @@ -7462,7 +7465,47 @@ fn overlay_staging_dir(overlay_disk: &Path) -> PathBuf { )) } +/// Grace after SIGTERM before killing a host process. +const PROCESS_TERMINATION_GRACE: Duration = Duration::from_secs(5); + +/// Mirrors the supervisor's workload SIGTERM grace. +const SUPERVISOR_WORKLOAD_TERM_SECS: u64 = 5; +/// Guest process-tree termination grace plus forced-kill reap time. +const GUEST_PROCESS_TREE_SECS: u64 = DEFAULT_STOP_TIMEOUT_SECS as u64 + 2; +/// Mirrors the guest's filesystem flush timeout. +const GUEST_FLUSH_SECS: u64 = 3; +/// Allow control round trips and scheduling between shutdown stages. +const SHUTDOWN_SCHEDULING_SECS: u64 = 2; + +/// Budget for workload termination, guest process cleanup and filesystem flush. +/// The cap may interrupt slow shutdown or log draining. Only the stop timeout +/// is shared; changes to the other stage limits must be reflected here. +const SUPERVISOR_SHUTDOWN_GRACE: Duration = Duration::from_secs( + SUPERVISOR_WORKLOAD_TERM_SECS + + GUEST_PROCESS_TREE_SECS + + GUEST_FLUSH_SECS + + SHUTDOWN_SCHEDULING_SECS, +); + +/// Mirrors the gateway's stop deadline when reclaiming expired provisioning. +const STOP_CALLER_DEADLINE: Duration = Duration::from_secs(30); + +// The longest stop is the supervisor's grace plus the VM's. +const _: () = assert!( + SUPERVISOR_SHUTDOWN_GRACE.as_secs() + PROCESS_TERMINATION_GRACE.as_secs() + < STOP_CALLER_DEADLINE.as_secs() +); + async fn terminate_vm_process(child: &mut Child) -> Result<(), std::io::Error> { + terminate_vm_process_within(child, PROCESS_TERMINATION_GRACE) + .await + .map(|_| ()) +} + +async fn terminate_vm_process_within( + child: &mut Child, + grace: Duration, +) -> Result { if let Some(pid) = child.id() && let Err(err) = kill(Pid::from_raw(pid.cast_signed()), Signal::SIGTERM) && err != Errno::ESRCH @@ -7472,23 +7515,41 @@ async fn terminate_vm_process(child: &mut Child) -> Result<(), std::io::Error> { ))); } - match tokio::time::timeout(Duration::from_secs(5), child.wait()).await { - Ok(Ok(_)) => Ok(()), - Ok(Err(err)) => Err(err), - Err(_) => { - child.kill().await?; - child.wait().await.map(|_| ()) - } + if let Ok(status) = tokio::time::timeout(grace, child.wait()).await { + status + } else { + child.kill().await?; + child.wait().await } } +/// Stop the host supervisor first, then the VM it was supervising. +/// +/// Keep liveness open until the supervisor exits: EOF skips orderly workload +/// teardown, the guest filesystem flush and the supervisor's log export drain. async fn terminate_sandbox_processes(process: &mut VmProcess) -> Result<(), std::io::Error> { - // Liveness EOF makes the supervisor exit at once, skipping workload - // teardown and its OCSF export flush, so close it only after SIGTERM has - // had its chance. Each process keeps its five-second budget, so a stop - // still completes within ten seconds; a slow workload stop can leave the - // supervisor too little time to flush. - let supervisor_error = terminate_vm_process(&mut process.supervisor).await.err(); + terminate_sandbox_processes_within(process, SUPERVISOR_SHUTDOWN_GRACE).await +} + +async fn terminate_sandbox_processes_within( + process: &mut VmProcess, + supervisor_grace: Duration, +) -> Result<(), std::io::Error> { + let supervisor_pid = process.supervisor.id(); + let supervisor_error = terminate_vm_process_within(&mut process.supervisor, supervisor_grace) + .await + .inspect(|status| { + // An orderly shutdown returns the workload's exit code, which can + // be nonzero even after the guest acknowledges termination. + if status.signal().is_some() { + warn!( + ?supervisor_pid, + %status, + "VM host supervisor was terminated by a signal; guest filesystem flush was not confirmed and recent writes may be lost" + ); + } + }) + .err(); process.supervisor_liveness.take(); let vm_error = terminate_vm_process(&mut process.child).await.err(); @@ -12008,57 +12069,184 @@ mod tests { .unwrap() } - /// A stand-in supervisor that drains for a second on SIGTERM and then - /// writes `marker`, unless liveness reached EOF while it drained, which - /// would have made the real supervisor exit at once. - #[cfg(unix)] - fn spawn_draining_supervisor(marker: &Path) -> (Child, fs::File) { - let (liveness_read, liveness_write) = nix::unistd::pipe().unwrap(); - // As in production, so the child does not inherit the write end. - nix::fcntl::fcntl( - liveness_write.as_raw_fd(), - nix::fcntl::FcntlArg::F_SETFD(nix::fcntl::FdFlag::FD_CLOEXEC), - ) - .unwrap(); - let mut child = Command::new("sh") + /// Delay SIGTERM handling so premature liveness EOF wins the race. + #[tokio::test] + async fn stop_signals_supervisor_before_closing_its_liveness_pipe() { + let dir = unique_temp_dir(); + fs::create_dir_all(&dir).unwrap(); + let driver_log = dir.join("driver.log"); + let subscriber = tracing_subscriber::fmt() + .with_max_level(tracing::Level::WARN) + .without_time() + .with_ansi(false) + .with_writer(Arc::new(fs::File::create(&driver_log).unwrap())) + .finish(); + let _tracing = tracing::subscriber::set_default(subscriber); + let events = dir.join("events"); + let (read_end, write_end) = nix::unistd::pipe().unwrap(); + for fd in [&read_end, &write_end] { + nix::fcntl::fcntl( + fd.as_raw_fd(), + nix::fcntl::FcntlArg::F_SETFD(nix::fcntl::FdFlag::FD_CLOEXEC), + ) + .unwrap(); + } + let read_fd = read_end.as_raw_fd(); + let mut command = Command::new("sh"); + command .arg("-c") .arg( - r#"(cat >/dev/null; touch "$0.eof") <&0 & -trap 'sleep 1; [ -e "$0.eof" ] || echo drained > "$0"; exit 0' TERM -wait $! -exit 1"#, + r#"events=$1 +(cat <&3 >/dev/null; echo eof >>"$events") & +trap 'sleep 0.2; echo term >>"$events"; exit 143' TERM +echo ready >>"$events" +while :; do sleep 0.05; done"#, ) - .arg(marker) - .stdin(Stdio::from(liveness_read)) + .arg("supervisor") + .arg(&events) + .stdin(Stdio::null()) .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .unwrap(); - // Let the shell install its trap before the test signals it. - std::thread::sleep(Duration::from_millis(200)); - assert!(child.try_wait().unwrap().is_none()); - (child, fs::File::from(liveness_write)) + .stderr(Stdio::null()); + // SAFETY: only async-signal-safe `dup2` runs between fork and exec. + unsafe { + command.pre_exec(move || { + if libc::dup2(read_fd, 3) < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + let supervisor = command.spawn().unwrap(); + drop(read_end); + + let ready = async { + while !fs::read_to_string(&events).is_ok_and(|log| log.contains("ready")) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }; + tokio::time::timeout(Duration::from_secs(5), ready) + .await + .expect("stand-in supervisor installs its SIGTERM handler"); + + let mut process = VmProcess { + child: spawn_exited_child(), + supervisor, + supervisor_liveness: Some(fs::File::from(write_end)), + deleting: false, + }; + terminate_sandbox_processes(&mut process).await.unwrap(); + + let log = fs::read_to_string(&events).unwrap(); + let first_event = log.lines().find(|line| *line != "ready"); + assert_eq!( + first_event, + Some("term"), + "SIGTERM must reach the supervisor before its liveness pipe closes; events: {log:?}" + ); + assert!( + fs::read_to_string(&driver_log).unwrap().is_empty(), + "an orderly supervisor shutdown must not warn about lost writes" + ); + fs::remove_dir_all(&dir).ok(); } - #[cfg(unix)] + /// A supervisor that ignores SIGTERM is killed once its grace expires, and + /// only then is its liveness pipe closed. #[tokio::test] - async fn stopping_a_sandbox_lets_the_supervisor_drain_before_closing_liveness() { + async fn stop_kills_a_supervisor_that_ignores_sigterm_after_the_grace() { let dir = unique_temp_dir(); fs::create_dir_all(&dir).unwrap(); - let marker = dir.join("drained"); - let (supervisor, liveness) = spawn_draining_supervisor(&marker); + let driver_log = dir.join("driver.log"); + let subscriber = tracing_subscriber::fmt() + .with_max_level(tracing::Level::WARN) + .without_time() + .with_ansi(false) + .with_writer(Arc::new(fs::File::create(&driver_log).unwrap())) + .finish(); + let _tracing = tracing::subscriber::set_default(subscriber); + let events = dir.join("events"); + let (read_end, write_end) = nix::unistd::pipe().unwrap(); + for fd in [&read_end, &write_end] { + nix::fcntl::fcntl( + fd.as_raw_fd(), + nix::fcntl::FcntlArg::F_SETFD(nix::fcntl::FdFlag::FD_CLOEXEC), + ) + .unwrap(); + } + let read_fd = read_end.as_raw_fd(); + let mut command = Command::new("sh"); + command + .arg("-c") + .arg( + r#"events=$1 +(cat <&3 >/dev/null; echo eof >>"$events") & +trap '' TERM +echo ready >>"$events" +while :; do sleep 0.05; done"#, + ) + .arg("supervisor") + .arg(&events) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + // SAFETY: only async-signal-safe `dup2` runs between fork and exec. + unsafe { + command.pre_exec(move || { + if libc::dup2(read_fd, 3) < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + let supervisor = command.spawn().unwrap(); + drop(read_end); + let ready = async { + while !fs::read_to_string(&events).is_ok_and(|log| log.contains("ready")) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }; + tokio::time::timeout(Duration::from_secs(5), ready) + .await + .expect("stand-in supervisor starts"); + let mut process = VmProcess { child: spawn_exited_child(), supervisor, - supervisor_liveness: Some(liveness), - deleting: true, + supervisor_liveness: Some(fs::File::from(write_end)), + deleting: false, }; + let grace = Duration::from_millis(300); + let started = tokio::time::Instant::now(); + terminate_sandbox_processes_within(&mut process, grace) + .await + .unwrap(); + let elapsed = started.elapsed(); - terminate_sandbox_processes(&mut process).await.unwrap(); - - assert_eq!(fs::read_to_string(&marker).unwrap(), "drained\n"); + assert!( + elapsed >= grace, + "the supervisor must get its full grace before the kill: {elapsed:?}" + ); + assert!( + process.supervisor.try_wait().unwrap().is_some(), + "the supervisor is gone once the grace has expired" + ); assert!(process.supervisor_liveness.is_none()); - let _ = fs::remove_dir_all(dir); + // The liveness pipe closes only after the supervisor has been dealt with. + let closed = async { + while !fs::read_to_string(&events).is_ok_and(|log| log.contains("eof")) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }; + tokio::time::timeout(Duration::from_secs(5), closed) + .await + .expect("the liveness pipe is closed after the stop"); + let driver_log = fs::read_to_string(&driver_log).unwrap(); + assert!( + driver_log.contains("WARN") + && driver_log.contains("guest filesystem flush was not confirmed"), + "a forced supervisor shutdown must warn that recent writes may be lost: {driver_log:?}" + ); + fs::remove_dir_all(&dir).ok(); } async fn insert_test_record( diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index a0c3bf5bc5..fee9dd3dbe 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -90,6 +90,9 @@ mod linux { const ENFORCEMENT_LOSS_TERMINATION_GRACE: Duration = Duration::from_secs(openshell_core::config::DEFAULT_STOP_TIMEOUT_SECS as u64); const FORCE_KILL_REAP_TIMEOUT: Duration = Duration::from_secs(2); + /// Bound the wait for a flush before acknowledging termination. A timed-out + /// blocking task cannot be cancelled and may outlive the acknowledgement. + const FILESYSTEM_FLUSH_TIMEOUT: Duration = Duration::from_secs(3); const MAX_PENDING_HANDSHAKES: usize = 32; const MAX_CONTROL_CONNECTIONS: usize = 128; const MAX_REPLAY_LEDGER_ENTRIES: usize = 4096; @@ -1426,6 +1429,42 @@ mod linux { Ok(()) } + /// Writes the sandbox's writable filesystem to its backing store. + type FlushFn = Arc io::Result<()> + Send + Sync>; + + /// Filesystem flush operation and its termination wait bound. + struct FilesystemFlush { + run: FlushFn, + timeout: Duration, + } + + impl FilesystemFlush { + /// VM page caches disappear at shutdown. Container page caches survive + /// in the host, where `syncfs` would flush the shared backing filesystem. + fn for_listener(listener: &BoundaryListenerConfig) -> Option { + match listener { + BoundaryListenerConfig::Vsock { .. } => Some(Self { + run: Arc::new(flush_root_filesystem), + timeout: FILESYSTEM_FLUSH_TIMEOUT, + }), + BoundaryListenerConfig::Unix { .. } | BoundaryListenerConfig::TlsTcp { .. } => None, + } + } + } + + /// Flush the filesystem mounted at `/`, which in the VM holds the sandbox's + /// writable layer. + fn flush_root_filesystem() -> io::Result<()> { + sync_filesystem_at(Path::new("/")) + } + + /// Write back the filesystem that contains `path`. `syncfs` is used instead + /// of a global `sync` so only that filesystem is flushed. + fn sync_filesystem_at(path: &Path) -> io::Result<()> { + let directory = File::open(path)?; + nix::unistd::syncfs(directory.as_raw_fd()).map_err(io::Error::from) + } + struct BoundaryRuntime { config: BoundaryConfig, routing_handle: String, @@ -1436,6 +1475,8 @@ mod linux { process_runtime: tokio::runtime::Handle, state: Mutex, supervisor_connection: Mutex, + /// VM-only termination flush. + filesystem_flush: Option, next_recovery_id: AtomicU64, /// The wire policy bound at first attach, so an idempotent attach retry /// carrying a different policy is denied instead of silently keeping @@ -1721,6 +1762,7 @@ mod linux { Arc::new(SystemJwtClock), ) .map_err(|error| format!("configure Sandbox Protocol JWT verifier: {error}"))?; + let filesystem_flush = FilesystemFlush::for_listener(&config.listener); Ok(Self { authenticator: SandboxProtocolAuthenticator::new( verifier, @@ -1738,6 +1780,7 @@ mod linux { process_runtime, state: Mutex::new(RuntimeState::AwaitingAttach), supervisor_connection: Mutex::new(SupervisorConnectionState::AwaitingConfirmation), + filesystem_flush, next_recovery_id: AtomicU64::new(1), attached_policy: Mutex::new(None), started_agent: Mutex::new(None), @@ -1753,6 +1796,12 @@ mod linux { }) } + #[cfg(test)] + fn with_filesystem_flush(mut self, filesystem_flush: FilesystemFlush) -> Self { + self.filesystem_flush = Some(filesystem_flush); + self + } + fn authenticate_request( &self, connection_id: SandboxConnectionId, @@ -2023,15 +2072,44 @@ mod linux { | RuntimeState::Ready(_) => None, } }; - if let Some(process) = process - && let Err(error) = Self::terminate_process_tree(&process, false).await - { + let terminated = match process { + Some(process) => Self::terminate_process_tree(&process, false).await, + None => Ok(()), + }; + // The host may end the VM on acknowledgement. Flush dirty writes + // first, even if process termination failed. + self.flush_filesystem().await; + if let Err(error) = terminated { return guest_error(BoundaryErrorKind::Process, error); } *lock(&self.supervisor_connection) = SupervisorConnectionState::Terminal; Response::BoundaryTerminated } + /// Log flush failures and timeouts while allowing termination to finish. + async fn flush_filesystem(&self) { + let Some(filesystem_flush) = &self.filesystem_flush else { + return; + }; + let run = Arc::clone(&filesystem_flush.run); + let flush = tokio::task::spawn_blocking(move || run()); + match tokio::time::timeout(filesystem_flush.timeout, flush).await { + Ok(Ok(Ok(()))) => {} + Ok(Ok(Err(error))) => { + tracing::warn!(%error, "Sandbox filesystem flush failed before termination"); + } + Ok(Err(error)) => { + tracing::warn!(%error, "Sandbox filesystem flush task failed before termination"); + } + Err(_) => { + tracing::warn!( + timeout = ?filesystem_flush.timeout, + "Sandbox filesystem flush timed out before termination" + ); + } + } + } + async fn terminate_process_tree( process: &ManagedProcess, enforcement_was_lost: bool, @@ -4325,37 +4403,52 @@ mod linux { } fn availability_test_runtime() -> (Arc, String) { + availability_test_runtime_with(None) + } + + fn availability_test_runtime_with_flush( + flush: FilesystemFlush, + ) -> (Arc, String) { + availability_test_runtime_with(Some(flush)) + } + + /// Build a runtime over a TCP listener. `None` keeps the flush that the + /// listener kind selects in production. + fn availability_test_runtime_with( + flush: Option, + ) -> (Arc, String) { let (broker, launcher) = test_network_broker(); let (verification_key, token) = test_auth_material("availability"); - let runtime = Arc::new( - BoundaryRuntime::new( - BoundaryConfig { - boundary_id: "availability".to_string(), - generation: "generation-1".to_string(), - session_id: test_session_id(), - session_rotation: openshell_core::jwt::SessionRotation::new(1) - .expect("session rotation"), - auth_epoch: CredentialEpoch::new(1).expect("auth epoch"), - gateway_id: "test-gateway".to_string(), - verification_keys: vec![verification_key], - listener: BoundaryListenerConfig::TlsTcp { - address: "127.0.0.1:5500".parse().unwrap(), - tls: placeholder_server_tls(), - }, - resource_claims: std::collections::BTreeMap::new(), - resource_claim_files: std::collections::BTreeMap::new(), - workload_identity: test_workload_identity(), - provisioning_evidence: test_provisioning_evidence(), - child_env: std::collections::HashMap::new(), + let mut runtime = BoundaryRuntime::new( + BoundaryConfig { + boundary_id: "availability".to_string(), + generation: "generation-1".to_string(), + session_id: test_session_id(), + session_rotation: openshell_core::jwt::SessionRotation::new(1) + .expect("session rotation"), + auth_epoch: CredentialEpoch::new(1).expect("auth epoch"), + gateway_id: "test-gateway".to_string(), + verification_keys: vec![verification_key], + listener: BoundaryListenerConfig::TlsTcp { + address: "127.0.0.1:5500".parse().unwrap(), + tls: placeholder_server_tls(), }, - tokio::runtime::Handle::current(), - broker, - launcher, - test_runtime_qualification(), - ) - .expect("test boundary runtime"), - ); - (runtime, token) + resource_claims: std::collections::BTreeMap::new(), + resource_claim_files: std::collections::BTreeMap::new(), + workload_identity: test_workload_identity(), + provisioning_evidence: test_provisioning_evidence(), + child_env: std::collections::HashMap::new(), + }, + tokio::runtime::Handle::current(), + broker, + launcher, + test_runtime_qualification(), + ) + .expect("test boundary runtime"); + if let Some(flush) = flush { + runtime = runtime.with_filesystem_flush(flush); + } + (Arc::new(runtime), token) } #[tokio::test] @@ -4548,6 +4641,165 @@ mod linux { ); } + #[tokio::test(flavor = "multi_thread")] + async fn termination_flushes_the_filesystem_before_the_terminal_acknowledgement() { + let runtime_cell: Arc>> = + Arc::default(); + let flushes = Arc::new(AtomicUsize::new(0)); + let saw_terminating = Arc::new(AtomicBool::new(false)); + let flush = { + let runtime_cell = Arc::clone(&runtime_cell); + let flushes = Arc::clone(&flushes); + let saw_terminating = Arc::clone(&saw_terminating); + FilesystemFlush { + run: Arc::new(move || { + // Flush before the terminal transition permits VM teardown. + if let Some(runtime) = runtime_cell.get().and_then(std::sync::Weak::upgrade) + { + saw_terminating.store( + matches!( + *lock(&runtime.supervisor_connection), + SupervisorConnectionState::Terminating + ), + Ordering::SeqCst, + ); + } + flushes.fetch_add(1, Ordering::SeqCst); + Ok(()) + }), + timeout: FILESYSTEM_FLUSH_TIMEOUT, + } + }; + let (runtime, _token) = availability_test_runtime_with_flush(flush); + runtime_cell + .set(Arc::downgrade(&runtime)) + .expect("runtime cell is set once"); + + assert_eq!( + runtime.terminate_boundary().await, + Response::BoundaryTerminated + ); + + assert_eq!(flushes.load(Ordering::SeqCst), 1); + assert!( + saw_terminating.load(Ordering::SeqCst), + "the flush must run before the boundary turns terminal" + ); + // A repeated request is answered from the terminal state and does not + // flush again. + assert_eq!( + runtime.terminate_boundary().await, + Response::BoundaryTerminated + ); + assert_eq!(flushes.load(Ordering::SeqCst), 1); + } + + #[test] + fn only_the_vm_listener_flushes_the_filesystem() { + let tls = placeholder_server_tls(); + let vsock = BoundaryListenerConfig::Vsock { + control_port: 5500, + tls: tls.clone(), + }; + let flush = FilesystemFlush::for_listener(&vsock).expect("the VM backend flushes"); + assert_eq!(flush.timeout, FILESYSTEM_FLUSH_TIMEOUT); + + let unix = BoundaryListenerConfig::Unix { + socket_path: "/run/openshell/control.sock".into(), + tls: tls.clone(), + }; + let tcp = BoundaryListenerConfig::TlsTcp { + address: "127.0.0.1:5500".parse().unwrap(), + tls, + }; + assert!( + FilesystemFlush::for_listener(&unix).is_none(), + "container sandboxes on a Unix socket must not flush" + ); + assert!( + FilesystemFlush::for_listener(&tcp).is_none(), + "container sandboxes on a TCP listener must not flush" + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn container_listener_termination_runs_no_flush() { + let (runtime, _token) = availability_test_runtime_with(None); + assert!( + runtime.filesystem_flush.is_none(), + "a TCP-listener boundary must not install a flush" + ); + + assert_eq!( + runtime.terminate_boundary().await, + Response::BoundaryTerminated + ); + assert_eq!( + *lock(&runtime.supervisor_connection), + SupervisorConnectionState::Terminal + ); + } + + /// Exercise the real syscall and open-error path; persistence requires + /// VM stop/start testing. + #[test] + fn real_filesystem_flush_syncs_an_existing_path_and_reports_a_missing_one() { + let directory = tempfile::tempdir().expect("temporary directory"); + std::fs::write(directory.path().join("data"), b"x").expect("write test data"); + sync_filesystem_at(directory.path()).expect("sync the filesystem holding the path"); + flush_root_filesystem().expect("sync the root filesystem"); + + let missing = directory.path().join("missing"); + let error = sync_filesystem_at(&missing).expect_err("a missing path cannot be synced"); + assert_eq!(error.kind(), io::ErrorKind::NotFound); + } + + #[tokio::test(flavor = "multi_thread")] + async fn termination_is_acknowledged_when_the_filesystem_flush_fails() { + let (runtime, _token) = availability_test_runtime_with_flush(FilesystemFlush { + run: Arc::new(|| Err(io::Error::other("flush failed"))), + timeout: FILESYSTEM_FLUSH_TIMEOUT, + }); + + assert_eq!( + runtime.terminate_boundary().await, + Response::BoundaryTerminated + ); + assert_eq!( + *lock(&runtime.supervisor_connection), + SupervisorConnectionState::Terminal + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn termination_is_acknowledged_when_the_filesystem_flush_hangs() { + let (release, blocked) = std::sync::mpsc::channel::<()>(); + let blocked = Mutex::new(blocked); + let (runtime, _token) = availability_test_runtime_with_flush(FilesystemFlush { + run: Arc::new(move || { + let _ = lock(&blocked).recv(); + Ok(()) + }), + timeout: Duration::from_millis(100), + }); + + let started = tokio::time::Instant::now(); + let response = runtime.terminate_boundary().await; + let elapsed = started.elapsed(); + // Let the blocked flush finish so the runtime can shut down. + release.send(()).expect("release the blocked flush"); + + assert_eq!(response, Response::BoundaryTerminated); + assert!( + elapsed < Duration::from_secs(2), + "a hung flush must not hold the acknowledgement: {elapsed:?}" + ); + assert_eq!( + *lock(&runtime.supervisor_connection), + SupervisorConnectionState::Terminal + ); + } + #[tokio::test(flavor = "multi_thread")] async fn explicit_shutdown_acknowledges_terminal_session_state() { let (runtime, token) = availability_test_runtime(); diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index 89ec2c8d44..f56d194d63 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -238,6 +238,12 @@ VM sandboxes have no network interface. All traffic flows through the OpenShell If the gateway is briefly unavailable while a VM starts, its supervisor retries the session connection. The sandbox stays in provisioning until the gateway accepts the session. +### MicroVM Stop Durability + +When you run `openshell sandbox stop` while the main process is running, OpenShell terminates the workload and flushes the guest root filesystem before acknowledging termination. The guest waits up to three seconds for the flush. The host allows up to 22 seconds for supervisor shutdown, then up to five seconds to stop the VM. + +Flush errors and timeouts produce warnings while shutdown continues. If a signal kills the supervisor, the driver warns that the guest flush was not confirmed and recent writes may be lost. A completed stop does not guarantee persistence after a failed or timed-out flush. This flush does not cover stops after the main process has already exited. + ### MicroVM Mounts The VM driver shares gateway-host directories into the guest with virtiofs. Each entry in `mounts` takes `source` (absolute host directory), `target`, optional `read_only` (default `true`), and optional `type`, which must be `bind` so Docker and Podman bind entries work unchanged: