diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index d924bf1ece..0b5a069eeb 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -68,15 +68,33 @@ gatewayConfig: supervisor_topology: gateway ``` -The gateway chart creates a separate, single-replica shared-host placeholder Deployment -in the release namespace, regardless of shared/managed/operator workspace mode. -Use a supervisor image built from this branch: `--mode=shared` is new. -The chart sets `--gateway-endpoint` from the gateway's gRPC endpoint and -`--address` to `0.0.0.0:5501`. Check its rollout and TCP health probes on this -listener (container port `supervisor`, 5501). The outbound gateway connection and -session RPCs are not implemented yet; sandboxes -still use dedicated supervisors. `workspace` is reserved and rejected. Omit the -field to disable it. +The gateway chart creates a single-replica shared supervisor Deployment, +Service, NetworkPolicy, and an empty `-supervisor-token` Secret in +the release namespace, regardless of shared/managed/operator workspace mode. +The gateway mints a supervisor token from its sandbox-JWT signing key, writes +it into that Secret (the gateway Role may `get`/`patch` only that Secret), and +renews it at half its lifetime; the supervisor mounts the Secret and presents +the token on the private `openshell.supervisor.host.v1` assignment stream, +which accepts nothing else. The topology requires gateway TLS and the signing +key (`server.disableTls` is rejected; the gateway refuses to start without +`gateway_jwt`). New sandbox generations run as sandbox instances in that +process. The chart starts `--mode=shared`, sets `--gateway-endpoint` to the +gateway gRPC URL, mounts the token at `--token-file`, and binds `--address` +(5501) to a plain TCP health listener for the `tcpSocket` probes; nothing +dials the supervisor but sandbox runtimes (5500) and the kubelet. Build the gateway, supervisor, and sandbox +images from the same branch: all three must support the routed runtime +connection and the assignment stream. `workspace` is reserved and rejected. +Omit the field to disable it. `mise run +e2e:kubernetes:shared-supervisor` runs the focused e2e +(`ci/values-supervisor.yaml`). The task prepends that overlay to any +colon-separated `OPENSHELL_E2E_KUBE_EXTRA_VALUES` list, so layer a +workspace-mode overlay to cover managed namespaces: + +```bash +OPENSHELL_E2E_KUBE_EXTRA_VALUES=deploy/helm/openshell/ci/values-workspace-managed.yaml \ +OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET=e2e-regcred \ + mise run e2e:kubernetes:shared-supervisor +``` ### 2. Deploy OpenShell diff --git a/AGENTS.md b/AGENTS.md index 8ab23c2a90..d1652c2606 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,6 +26,7 @@ Do not rely on this file for a full inventory. The detailed public and contribut | `crates/openshell-server/` | Gateway server | Control-plane API, sandbox lifecycle, auth boundary | | `crates/openshell-sandbox/` | Sandbox runtime | Capability-free workload launcher, process identity, and seccomp-mediated I/O | | `crates/openshell-supervisor/` | Supervisor runtime | Gateway session, policy evaluation, credentials, and upstream networking | +| `crates/openshell-supervisor-control/` | Shared supervisor control | Gateway-owned assignment controller, launch registrations, recovery coordination, and host observations for a supervisor shared by many sandboxes | | `crates/openshell-binary-identity/` | Binary identity | Shared trusted procfs executable identity resolution for isolation backends | | `crates/openshell-isolation-interface/` | Isolation backend interface | Shared data and validation for the Isolation Backend RPC contracts | | `crates/openshell-sandbox-backend/` | OpenShell sandbox backend | Concrete Isolation Backend RPC client and authenticated Sandbox Protocol shared with `openshell-sandbox` | diff --git a/Cargo.lock b/Cargo.lock index 2b00e6e6c5..f36af0eb0f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4505,16 +4505,19 @@ dependencies = [ "openshell-otel-test-support", "openshell-policy", "openshell-sandbox-backend", + "openshell-supervisor-control", "opentelemetry", "opentelemetry_sdk", "prost", "prost-types", "rcgen", + "rustls", "serde", "serde_json", "temp-env", "thiserror 2.0.20", "tokio", + "tokio-rustls", "tokio-stream", "toml", "tonic", @@ -5028,6 +5031,7 @@ dependencies = [ "openshell-policy", "openshell-prover", "openshell-providers", + "openshell-supervisor-control", "openshell-supervisor-middleware", "openshell-supervisor-middleware-builtins", "opentelemetry", @@ -5098,6 +5102,7 @@ dependencies = [ "openshell-otel", "openshell-policy", "openshell-sandbox-backend", + "openshell-supervisor-control", "openshell-supervisor-middleware", "openshell-supervisor-middleware-builtins", "openshell-supervisor-network", @@ -5114,6 +5119,8 @@ dependencies = [ "tempfile", "thiserror 2.0.20", "tokio", + "tokio-rustls", + "tokio-stream", "tokio-tungstenite 0.26.2", "tonic", "tonic-prost", @@ -5123,6 +5130,23 @@ dependencies = [ "uuid", ] +[[package]] +name = "openshell-supervisor-control" +version = "0.0.0" +dependencies = [ + "openshell-core", + "openshell-otel", + "openshell-sandbox-backend", + "prost-types", + "serde", + "serde_json", + "thiserror 2.0.20", + "tokio", + "tokio-stream", + "tonic", + "tracing", +] + [[package]] name = "openshell-supervisor-middleware" version = "0.0.0" diff --git a/crates/openshell-core/src/grpc_client.rs b/crates/openshell-core/src/grpc_client.rs index 75672d0263..3a13ed88e5 100644 --- a/crates/openshell-core/src/grpc_client.rs +++ b/crates/openshell-core/src/grpc_client.rs @@ -564,6 +564,15 @@ pub async fn connect_channel_pub(gateway: &GatewayEndpoint) -> Result Result { + build_plain_channel(endpoint).await +} + /// Report installed provider state for the current authenticated supervisor session. /// /// The observation must carry the session ID returned by `ConnectSupervisor`. diff --git a/crates/openshell-core/src/jwt.rs b/crates/openshell-core/src/jwt.rs index c6b3d1b33a..7b1bf57ffd 100644 --- a/crates/openshell-core/src/jwt.rs +++ b/crates/openshell-core/src/jwt.rs @@ -47,6 +47,9 @@ mod session { pub const GATEWAY_SESSION_JWT_TYPE: &str = "openshell-gateway-session+jwt"; pub const SANDBOX_SESSION_JWT_TYPE: &str = "openshell-sandbox-session+jwt"; + /// JOSE `typ` of the supervisor token: the credential of the gateway's + /// shared supervisor process for the assignment stream, and nothing else. + pub const SUPERVISOR_JWT_TYPE: &str = "openshell-supervisor+jwt"; pub const SANDBOX_SESSION_AUDIENCE: &str = "openshell-sandbox"; pub const DEFAULT_SESSION_TOKEN_TTL: Duration = Duration::from_hours(1); pub const MIN_SESSION_TOKEN_TTL: Duration = Duration::from_mins(1); @@ -55,6 +58,7 @@ mod session { const GATEWAY_ISSUER_PREFIX: &str = "openshell-gateway:"; const SANDBOX_SUBJECT_PREFIX: &str = "spiffe://openshell/sandbox/"; + const SUPERVISOR_SUBJECT_PREFIX: &str = "openshell-supervisor:"; /// Canonical sandbox identity carried by both session-token profiles. #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] @@ -175,6 +179,21 @@ mod session { component: SessionComponent, } + /// Claims of the supervisor token. It names no sandbox, generation, or + /// credential epoch: the shared supervisor acts for sandboxes only with the + /// per-instance credentials each assignment carries. + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] + #[serde(deny_unknown_fields)] + struct SupervisorClaims { + iss: String, + sub: String, + aud: String, + iat: i64, + exp: i64, + jti: String, + component: SessionComponent, + } + /// Durable identity shared by every short-lived token for one sandbox runtime. #[derive(Clone, Debug, PartialEq, Eq)] pub struct SandboxRuntimeIdentity { @@ -668,6 +687,40 @@ mod session { }) } + /// Mint the shared supervisor's token for the gateway's assignment + /// stream with its own bounded lifetime, independent of the session + /// token lifetime (which may be non-expiring). + pub fn mint_supervisor_token( + &self, + ttl: Duration, + ) -> Result { + validate_ttl(ttl)?; + let issued_at = self.clock.now_unix_seconds(); + let expires_at = + issued_at.saturating_add(i64::try_from(ttl.as_secs()).unwrap_or(i64::MAX)); + let token_id = Uuid::new_v4(); + let gateway_id = &self.issuer[GATEWAY_ISSUER_PREFIX.len()..]; + let claims = SupervisorClaims { + iss: self.issuer.clone(), + sub: format!("{SUPERVISOR_SUBJECT_PREFIX}{gateway_id}"), + aud: self.issuer.clone(), + iat: issued_at, + exp: expires_at, + jti: token_id.to_string(), + component: SessionComponent::OpenShellSupervisor, + }; + let mut header = Header::new(Algorithm::EdDSA); + header.kid = Some(self.key_id.clone()); + header.typ = Some(SUPERVISOR_JWT_TYPE.to_string()); + let token = encode(&header, &claims, &self.encoding_key) + .map_err(|_| SessionJwtError::SigningFailed)?; + Ok(MintedSessionToken { + token: SecretJwt::parse(token)?, + expires_at, + token_id, + }) + } + fn mint( &self, profile: SessionTokenProfile, @@ -749,20 +802,8 @@ mod session { ) -> Result { install_crypto_provider(); let gateway_id = validate_gateway_id(gateway_id)?; - let mut parsed = BTreeMap::new(); - for key in keys { - let key_id = validate_key_id(key.key_id)?; - let decoding_key = DecodingKey::from_ed_pem(&key.public_key_pem) - .map_err(|_| SessionJwtError::InvalidVerificationKey)?; - if parsed.insert(key_id, decoding_key).is_some() { - return Err(SessionJwtError::DuplicateKeyId); - } - } - if parsed.is_empty() { - return Err(SessionJwtError::NoVerificationKeys); - } Ok(Self { - keys: parsed, + keys: parse_verification_keys(keys)?, issuer: format!("{GATEWAY_ISSUER_PREFIX}{gateway_id}"), profile, clock, @@ -770,27 +811,12 @@ mod session { } pub fn verify(&self, token: &str) -> Result { - install_crypto_provider(); - let header = decode_header(token).map_err(|_| SessionJwtError::InvalidToken)?; - if header.alg != Algorithm::EdDSA { - return Err(SessionJwtError::WrongAlgorithm); - } - if header.typ.as_deref() != Some(self.profile.token_type()) { - return Err(SessionJwtError::WrongTokenType); - } - let key_id = header.kid.ok_or(SessionJwtError::MissingKeyId)?; - let key = self - .keys - .get(&key_id) - .ok_or(SessionJwtError::UnknownKeyId)?; - let mut validation = Validation::new(Algorithm::EdDSA); - validation.algorithms = vec![Algorithm::EdDSA]; - validation.validate_exp = false; - validation.validate_aud = false; - validation.set_required_spec_claims(&["iss", "aud", "iat", "exp", "sub", "jti"]); - let claims = decode::(token, key, &validation) - .map_err(|_| SessionJwtError::InvalidToken)? - .claims; + let claims = decode_signed::( + &self.keys, + token, + self.profile.token_type(), + &["iss", "aud", "iat", "exp", "sub", "jti"], + )?; self.validate_claims(claims) } @@ -810,23 +836,7 @@ mod session { SandboxGenerationId::parse(claims.runtime_generation.to_string()) .map_err(|_| SessionJwtError::InvalidRuntimeIdentity)?; let token_id = Uuid::parse_str(&claims.jti).map_err(|_| SessionJwtError::InvalidJti)?; - let now = self.clock.now_unix_seconds(); - let leeway = i64::try_from(MAX_SESSION_CLOCK_LEEWAY.as_secs()).unwrap_or(30); - if claims.iat > now.saturating_add(leeway) { - return Err(SessionJwtError::IssuedInFuture); - } - if claims.exp != 0 { - if claims.exp <= claims.iat { - return Err(SessionJwtError::InvalidLifetime); - } - let lifetime = claims.exp.saturating_sub(claims.iat); - if lifetime > i64::try_from(MAX_SESSION_TOKEN_TTL.as_secs()).unwrap_or(i64::MAX) { - return Err(SessionJwtError::InvalidLifetime); - } - if claims.exp < now.saturating_sub(leeway) { - return Err(SessionJwtError::Expired); - } - } + validate_lifetime(claims.iat, claims.exp, self.clock.now_unix_seconds())?; Ok(AuthenticatedSandboxSession { sandbox_id: claims.sandbox_id, runtime_generation: claims.runtime_generation, @@ -838,6 +848,151 @@ mod session { } } + /// A verified supervisor token. + #[derive(Clone, Debug, PartialEq, Eq)] + pub struct AuthenticatedSupervisor { + /// `openshell-supervisor:`. + pub subject: String, + pub token_id: Uuid, + pub issued_at: i64, + pub expires_at: i64, + } + + /// Strict gateway-side verifier of supervisor tokens. Accepts only the + /// supervisor profile signed by this gateway for this gateway. + pub struct SupervisorJwtVerifier { + keys: BTreeMap, + issuer: String, + subject: String, + clock: Arc, + } + + impl fmt::Debug for SupervisorJwtVerifier { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("SupervisorJwtVerifier") + .field("key_ids", &self.keys.keys().collect::>()) + .field("issuer", &self.issuer) + .finish_non_exhaustive() + } + } + + impl SupervisorJwtVerifier { + pub fn new( + gateway_id: &str, + keys: impl IntoIterator, + clock: Arc, + ) -> Result { + install_crypto_provider(); + let gateway_id = validate_gateway_id(gateway_id)?; + Ok(Self { + keys: parse_verification_keys(keys)?, + issuer: format!("{GATEWAY_ISSUER_PREFIX}{gateway_id}"), + subject: format!("{SUPERVISOR_SUBJECT_PREFIX}{gateway_id}"), + clock, + }) + } + + pub fn verify(&self, token: &str) -> Result { + let claims = decode_signed::( + &self.keys, + token, + SUPERVISOR_JWT_TYPE, + &["iss", "aud", "iat", "exp", "sub", "jti"], + )?; + if claims.iss != self.issuer { + return Err(SessionJwtError::WrongIssuer); + } + if claims.aud != self.issuer { + return Err(SessionJwtError::WrongAudience); + } + if claims.sub != self.subject { + return Err(SessionJwtError::SubjectMismatch); + } + let token_id = Uuid::parse_str(&claims.jti).map_err(|_| SessionJwtError::InvalidJti)?; + // Supervisor tokens must expire; a non-expiring one is refused. + if claims.exp == 0 { + return Err(SessionJwtError::InvalidLifetime); + } + validate_lifetime(claims.iat, claims.exp, self.clock.now_unix_seconds())?; + Ok(AuthenticatedSupervisor { + subject: claims.sub, + token_id, + issued_at: claims.iat, + expires_at: claims.exp, + }) + } + } + + fn parse_verification_keys( + keys: impl IntoIterator, + ) -> Result, SessionJwtError> { + let mut parsed = BTreeMap::new(); + for key in keys { + let key_id = validate_key_id(key.key_id)?; + let decoding_key = DecodingKey::from_ed_pem(&key.public_key_pem) + .map_err(|_| SessionJwtError::InvalidVerificationKey)?; + if parsed.insert(key_id, decoding_key).is_some() { + return Err(SessionJwtError::DuplicateKeyId); + } + } + if parsed.is_empty() { + return Err(SessionJwtError::NoVerificationKeys); + } + Ok(parsed) + } + + /// Check the header (`EdDSA` only, exact `typ`, known `kid`) and the + /// signature, then decode the claims. Time and audience are validated by + /// the caller against its own clock and profile. + fn decode_signed( + keys: &BTreeMap, + token: &str, + token_type: &str, + required_claims: &[&str], + ) -> Result { + install_crypto_provider(); + let header = decode_header(token).map_err(|_| SessionJwtError::InvalidToken)?; + if header.alg != Algorithm::EdDSA { + return Err(SessionJwtError::WrongAlgorithm); + } + if header.typ.as_deref() != Some(token_type) { + return Err(SessionJwtError::WrongTokenType); + } + let key_id = header.kid.ok_or(SessionJwtError::MissingKeyId)?; + let key = keys.get(&key_id).ok_or(SessionJwtError::UnknownKeyId)?; + let mut validation = Validation::new(Algorithm::EdDSA); + validation.algorithms = vec![Algorithm::EdDSA]; + validation.validate_exp = false; + validation.validate_aud = false; + validation.set_required_spec_claims(required_claims); + Ok(decode::(token, key, &validation) + .map_err(|_| SessionJwtError::InvalidToken)? + .claims) + } + + /// Shared lifetime rules: not issued in the future, and when expiring, + /// a positive lifetime within the session bound that has not passed. + fn validate_lifetime(issued_at: i64, expires_at: i64, now: i64) -> Result<(), SessionJwtError> { + let leeway = i64::try_from(MAX_SESSION_CLOCK_LEEWAY.as_secs()).unwrap_or(30); + if issued_at > now.saturating_add(leeway) { + return Err(SessionJwtError::IssuedInFuture); + } + if expires_at != 0 { + if expires_at <= issued_at { + return Err(SessionJwtError::InvalidLifetime); + } + let lifetime = expires_at.saturating_sub(issued_at); + if lifetime > i64::try_from(MAX_SESSION_TOKEN_TTL.as_secs()).unwrap_or(i64::MAX) { + return Err(SessionJwtError::InvalidLifetime); + } + if expires_at < now.saturating_sub(leeway) { + return Err(SessionJwtError::Expired); + } + } + Ok(()) + } + #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] pub enum SessionJwtError { #[error("sandbox ID is invalid")] @@ -1073,6 +1228,138 @@ mod tests { ); } + #[test] + fn supervisor_tokens_are_a_separate_profile_bound_to_the_gateway() { + let (issuer, gateway, sandbox, _identity) = fixture(); + let minted = issuer + .mint_supervisor_token(std::time::Duration::from_mins(15)) + .expect("supervisor token"); + assert_eq!(minted.expires_at, 1_900_000_000 + 900); + + let key = openshell_crypto::pki::generate_keypair_for(&PKCS_ED25519).unwrap(); + let public_key_pem = key.public_key_pem().into_bytes(); + let clock: Arc = Arc::new(FixedClock(1_900_000_000)); + // A verifier holding a different key under the same key ID sees a + // signature failure, not a key lookup failure. + let verification_key = |pem: Vec| SessionVerificationKey { + key_id: "current".to_string(), + public_key_pem: pem, + }; + let wrong_key = SupervisorJwtVerifier::new( + "test", + [verification_key(public_key_pem)], + clock.clone(), + ) + .expect("verifier"); + assert_eq!( + wrong_key.verify(minted.token.expose_secret()), + Err(SessionJwtError::InvalidToken) + ); + + // Session verifiers refuse the supervisor profile and the supervisor + // verifier refuses session tokens, whatever key signed them. + assert_eq!( + gateway.verify(minted.token.expose_secret()), + Err(SessionJwtError::WrongTokenType) + ); + assert_eq!( + sandbox.verify(minted.token.expose_secret()), + Err(SessionJwtError::WrongTokenType) + ); + let identity = SandboxRuntimeIdentity { + sandbox_id: SandboxId::parse("sandbox-a").unwrap(), + runtime_generation: SandboxGenerationId::parse("generation-1").unwrap(), + auth_epoch: CredentialEpoch::new(1).unwrap(), + }; + let pair = issuer.mint_pair(&identity).unwrap(); + assert_eq!( + wrong_key.verify(pair.gateway.token.expose_secret()), + Err(SessionJwtError::WrongTokenType) + ); + + // The supervisor lifetime is bounded on its own, whatever the + // session lifetime is. + drop(clock); + for ttl in [ + std::time::Duration::from_secs(59), + std::time::Duration::from_secs(3_601), + ] { + assert_eq!( + issuer.mint_supervisor_token(ttl).unwrap_err(), + SessionJwtError::InvalidLifetime + ); + } + } + + #[test] + fn supervisor_tokens_verify_only_for_their_gateway_and_lifetime() { + let key = openshell_crypto::pki::generate_keypair_for(&PKCS_ED25519).unwrap(); + let public_key_pem = key.public_key_pem().into_bytes(); + let signing_pem = key.serialize_pem().unwrap(); + let verification_key = || SessionVerificationKey { + key_id: "current".to_string(), + public_key_pem: public_key_pem.clone(), + }; + let issuer = |gateway_id: &str, now: i64| { + SessionJwtIssuer::from_ed25519_pem( + signing_pem.as_bytes(), + "current", + gateway_id, + Some(std::time::Duration::from_mins(15)), + Arc::new(FixedClock(now)), + ) + .unwrap() + }; + let verifier = |gateway_id: &str, now: i64| { + SupervisorJwtVerifier::new( + gateway_id, + [verification_key()], + Arc::new(FixedClock(now)), + ) + .unwrap() + }; + + let minted = issuer("gw-a", 1_900_000_000) + .mint_supervisor_token(std::time::Duration::from_mins(15)) + .unwrap(); + let authenticated = verifier("gw-a", 1_900_000_100) + .verify(minted.token.expose_secret()) + .expect("valid supervisor token"); + assert_eq!(authenticated.subject, "openshell-supervisor:gw-a"); + assert_eq!(authenticated.token_id, minted.token_id); + assert_eq!(authenticated.expires_at, 1_900_000_900); + + // Another gateway's token is refused by issuer, audience, and subject. + assert_eq!( + verifier("gw-b", 1_900_000_100).verify(minted.token.expose_secret()), + Err(SessionJwtError::WrongIssuer) + ); + // Expired tokens are refused past the leeway. + assert_eq!( + verifier("gw-a", 1_900_001_000).verify(minted.token.expose_secret()), + Err(SessionJwtError::Expired) + ); + assert_eq!( + verifier("gw-a", 1_899_999_000).verify(minted.token.expose_secret()), + Err(SessionJwtError::IssuedInFuture) + ); + assert_eq!( + verifier("gw-a", 1_900_000_100).verify("not-a-token"), + Err(SessionJwtError::InvalidToken) + ); + } + + #[derive(Serialize)] + struct NonExpiringSupervisor<'a> { + iss: &'a str, + sub: &'a str, + aud: &'a str, + iat: i64, + exp: i64, + jti: String, + component: SessionComponent, + } + #[test] fn non_expiring_session_tokens_remain_usable() { let key = openshell_crypto::pki::generate_keypair_for(&PKCS_ED25519) @@ -1110,6 +1397,46 @@ mod tests { .verify(pair.sandbox.token.expose_secret()) .expect("non-expiring token remains valid"); + // The default (non-expiring) session issuer still mints expiring + // supervisor tokens, and a supervisor token must expire. + let supervisor = issuer + .mint_supervisor_token(std::time::Duration::from_mins(15)) + .expect("supervisor token from a non-expiring issuer"); + assert_eq!(supervisor.expires_at, 1_900_000_000 + 900); + let supervisor_verifier = SupervisorJwtVerifier::new( + "test", + [SessionVerificationKey { + key_id: "current".to_string(), + public_key_pem: key.public_key_pem().into_bytes(), + }], + Arc::new(FixedClock(1_900_000_100)), + ) + .unwrap(); + supervisor_verifier + .verify(supervisor.token.expose_secret()) + .expect("expiring supervisor token"); + let mut header = Header::new(Algorithm::EdDSA); + header.kid = Some("current".to_string()); + header.typ = Some(SUPERVISOR_JWT_TYPE.to_string()); + let forever = encode( + &header, + &NonExpiringSupervisor { + iss: "openshell-gateway:test", + sub: "openshell-supervisor:test", + aud: "openshell-gateway:test", + iat: 1_900_000_000, + exp: 0, + jti: uuid::Uuid::new_v4().to_string(), + component: SessionComponent::OpenShellSupervisor, + }, + &EncodingKey::from_ed_pem(key.serialize_pem().unwrap().as_bytes()).unwrap(), + ) + .unwrap(); + assert_eq!( + supervisor_verifier.verify(&forever), + Err(SessionJwtError::InvalidLifetime) + ); + let slot = SessionBearerTokenSlot::new( pair.sandbox.token.clone(), pair.sandbox.expires_at, diff --git a/crates/openshell-core/src/proto/mod.rs b/crates/openshell-core/src/proto/mod.rs index 194a69e44d..fb200cadac 100644 --- a/crates/openshell-core/src/proto/mod.rs +++ b/crates/openshell-core/src/proto/mod.rs @@ -82,6 +82,11 @@ pub mod gateway_interceptor { pub use super::generated::openshell::gateway_interceptor::v1; } +/// Private gateway-to-shared-supervisor assignment protocol. +pub mod supervisor_host { + pub use super::generated::openshell::supervisor::host::v1; +} + pub use datamodel::v1::*; pub use gateway_interceptor::v1::*; pub use middleware::v1::*; diff --git a/crates/openshell-driver-kubernetes/Cargo.toml b/crates/openshell-driver-kubernetes/Cargo.toml index 9da035039c..f1d55cc0f0 100644 --- a/crates/openshell-driver-kubernetes/Cargo.toml +++ b/crates/openshell-driver-kubernetes/Cargo.toml @@ -16,9 +16,10 @@ path = "src/main.rs" [dependencies] openshell-crypto = { workspace = true, features = ["tls-kube"] } -openshell-core = { path = "../openshell-core", default-features = false } +openshell-core = { path = "../openshell-core", default-features = false, features = ["jwt"] } openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-sandbox-backend = { path = "../openshell-sandbox-backend" } +openshell-supervisor-control = { path = "../openshell-supervisor-control" } openshell-otel = { path = "../openshell-otel" } openshell-policy = { path = "../openshell-policy" } @@ -42,15 +43,17 @@ thiserror = { workspace = true } miette = { workspace = true } notify = "8" rcgen = { workspace = true } - -[dev-dependencies] -openshell-otel-test-support = { path = "../openshell-otel-test-support" } -opentelemetry_sdk = { workspace = true, features = ["testing"] } bytes = { workspace = true } http = { workspace = true } http-body-util = { workspace = true } hyper = { workspace = true } hyper-util = { workspace = true } +rustls = { workspace = true } +tokio-rustls = { workspace = true } + +[dev-dependencies] +openshell-otel-test-support = { path = "../openshell-otel-test-support" } +opentelemetry_sdk = { workspace = true, features = ["testing"] } temp-env = "0.3" tokio = { workspace = true, features = ["test-util"] } toml = { workspace = true } diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 9a9d6dca18..7dc2478d34 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -179,6 +179,90 @@ must not create permissive policies, create Pods, read bootstrap Secrets, or spoof the OpenShell role labels. Exact supervisor-to-sandbox authorization is still enforced by TLS, JWT claims, session generation, and recorded Pod UIDs. +### Shared supervisor topology + +With `supervisor_topology = "gateway"`, each new runtime +generation becomes a sandbox instance on one supervisor process that Helm +deploys in the gateway namespace, instead of a supervisor Pod and boundary +Service of its own. A sandbox instance is one running generation of a stable +Sandbox resource. The generation's placement is recorded on the Sandbox +resource (`openshell.ai/sandbox-runtime-supervisor-topology: gateway` and the +instance route in `openshell.ai/sandbox-runtime-supervisor-session`, whose key +keeps its established name) when the generation starts. Every later lifecycle decision reads the record, so changing the setting +never reinterprets a running generation, and an unknown or incomplete record +fails closed. + +- **Ownership.** The gateway owns which instances the shared supervisor + hosts. It constructs an `openshell_supervisor_control::SharedSupervision` + when the gateway binary builds this driver with + `KubernetesComputeDriver::with_shared_supervision`, serves its assignments + to the supervisor over the private `openshell.supervisor.host.v1` stream, + and mints the supervisor token the supervisor dials it with over TLS (see + the `openshell-supervisor` README); that token authorizes only the + assignment stream. The coordinator owns the control contracts (launch + material is persisted before an instance is assigned, recovery is + compare-and-replace against a baseline read before the list, an incomplete + recovery never replaces the published set, host observations are + interpreted once); this driver adapts them to Kubernetes objects. The + standalone driver binary cannot host the topology and refuses a + configuration that selects it. +- **Supervisor token delivery.** The gateway renews the token at half its + lifetime (the `gateway_jwt` extension-token TTL, 15 minutes by default, + bounded to one hour) and the coordinator retries a failed delivery until it + succeeds or a newer token supersedes it. The driver's part is the write: it + replaces the `token` key of the chart-created + `shared_supervisor.token_secret_name` Secret in `service_namespace`, which + the supervisor Pod mounts, and names a missing Secret or a denied patch in + the retried error. +- **Create and start.** The driver creates a per-sandbox egress allowance + `os-shared-control-` from the workload Pod to the Pods the shared + supervisor's Service selects, on the runtime port, binds its UID and + resourceVersion into the boundary evidence, and builds the instance's + registration (route, sandbox name, main process spec and agent image from + the Sandbox resource, the provisioned runtime descriptor, the generation's + launch credentials, its proxy CA, and the current trace context). The + coordinator validates and encodes it, persists the launch material through + the driver into the generation's `os-supervisor--` Secret (key + `instance.json`, owned by the workload Pod), and then assigns the instance + before the driver releases the workload Pod. Assignment does not wait for + the supervisor, so create still returns before the gateway persists the + generation. The runtime identity's last component is the instance route, + which no Pod UID can equal, so `authenticate_sandbox` cannot vouch for a + shared generation. +- **Readiness.** A shared generation is available while the coordinator + reports its instance as supervised and accepted by the gateway and the + driver finds its fence and egress allowance. An instance the supervisor has + reconciled past and does not host, hosts for another sandbox, or reports as + ended is unavailable, and reconciliation suspends the workload. A + disconnected supervisor is unknown and a supervisor that has not yet + reconciled to the assignment is degraded; neither suspends anything. + Bootstrap completion waits on the same verdict within the bootstrap grace. +- **Stop and delete.** The driver withdraws the instance before suspending or + deleting the workload, and the generation cleanup deletes its launch + Secret. Every reconciliation also recovers the desired set from the Sandbox + resources that may run: the driver begins the recovery before it lists + them, lists the shared generations that may run with their routes, and + reads each launch Secret's bytes on request; the coordinator reuses current + assignments, decodes and binds persisted material to the recorded route and + sandbox, keeps the previous set when any read fails, and refuses a list + that raced a lifecycle edit. Instances of deleted Sandboxes and of stops + made while the gateway was down are withdrawn, and a restarted gateway + recovers its assignments before it sends anything. +- **Isolation.** The supervisor token is a profile of its own, separate from + every instance's credentials; the gateway refuses it on every other RPC and + refuses every other identity on the assignment stream, and ends a stream + when the token that opened it expires. Sandboxes cannot reach the stream, + and the routing preface only selects which generation-pinned TLS identity + the supervisor demands. + +The topology does not support upstream proxies or provider SPIFFE identity, and +the shared supervisor is a single point of failure for the sandboxes it hosts. +Shared sandbox instances export no supervisor OTLP traces or OCSF events; the driver +warns at startup when the gateway configures that export. +When the setting is removed, reconciliation suspends shared generations, and a +restart creates the dedicated generation's boundary Service the shared one never +had. + ## Sandbox Resource The driver works with the `agents.x-k8s.io` `Sandbox` custom resource. It diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index ccf130fab6..598e0b20ce 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -202,12 +202,80 @@ pub enum SupervisorTopology { Workspace, } +/// The supervisor Helm deploys for the gateway topology. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SharedSupervisorConfig { + /// Service selecting the shared supervisor Pod. + pub service_name: String, + pub service_namespace: String, + /// Service port for runtime-initiated Sandbox Protocol connections. + pub runtime_port: u16, + /// Secret in `service_namespace` the driver keeps current with the + /// gateway-minted supervisor token (key `token`); the supervisor Pod + /// mounts it and presents the token on its assignment stream. + pub token_secret_name: String, +} + +impl SharedSupervisorConfig { + fn validate(&self) -> Result<(), String> { + if !is_dns_label(&self.service_name) || !is_dns_label(&self.service_namespace) { + return Err( + "shared_supervisor service_name and service_namespace must be DNS labels".into(), + ); + } + if self.runtime_port == 0 { + return Err("shared_supervisor runtime_port must be nonzero".into()); + } + if !is_dns_subdomain(&self.token_secret_name) { + return Err("shared_supervisor token_secret_name must be a DNS subdomain name".into()); + } + Ok(()) + } + + /// DNS name of the shared supervisor Service, which sandbox runtimes dial. + #[must_use] + pub fn service_host(&self) -> String { + format!("{}.{}.svc", self.service_name, self.service_namespace) + } +} + +/// RFC 1123 subdomain, as Kubernetes object names require. +fn is_dns_subdomain(value: &str) -> bool { + !value.is_empty() + && value.len() <= 253 + && value.split('.').all(|label| { + !label.is_empty() + && label.len() <= 63 + && label + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + && !label.starts_with('-') + && !label.ends_with('-') + }) +} + +fn is_dns_label(value: &str) -> bool { + !value.is_empty() + && value.len() <= 63 + && value + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + && !value.starts_with('-') + && !value.ends_with('-') +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct KubernetesComputeConfig { - /// Deploy an idle shared supervisor through Helm. Does not route sessions. + /// Supervise new sandbox generations with one supervisor shared by the + /// gateway instead of a supervisor Pod per sandbox. Existing generations + /// keep the topology recorded when they started. #[serde(skip_serializing_if = "Option::is_none")] pub supervisor_topology: Option, + /// Shared supervisor connection rendered by Helm with the gateway topology. + #[serde(skip_serializing_if = "Option::is_none")] + pub shared_supervisor: Option, /// Permit caller-supplied driver JSON. Does not waive resource admission. pub allow_driver_config: bool, /// Operator-owned external attachment approval policy. @@ -401,6 +469,7 @@ impl Default for KubernetesComputeConfig { fn default() -> Self { Self { supervisor_topology: None, + shared_supervisor: None, workspace_mode: WorkspaceMode::default(), allow_driver_config: false, resource_admission: @@ -454,9 +523,7 @@ impl Default for KubernetesComputeConfig { impl KubernetesComputeConfig { /// Validate startup configuration without connecting to Kubernetes. pub fn validate_configuration(&self) -> Result<(), String> { - if self.supervisor_topology == Some(SupervisorTopology::Workspace) { - return Err("supervisor_topology=workspace is not implemented".into()); - } + self.validate_supervisor_topology()?; self.validate_workspace_mode()?; self.validate_provider_spiffe_workload_api_socket_path()?; self.validate_sandbox_identity_config()?; @@ -466,6 +533,42 @@ impl KubernetesComputeConfig { self.validate_allowed_pod_labels() } + /// The shared supervisor serves only sandbox instances it can isolate: it + /// holds no per-sandbox workload identity, upstream proxy credential, or + /// proxy CA. + pub fn validate_supervisor_topology(&self) -> Result<(), String> { + match self.supervisor_topology { + None => { + if self.shared_supervisor.is_some() { + return Err("shared_supervisor requires supervisor_topology=gateway".into()); + } + Ok(()) + } + Some(SupervisorTopology::Workspace) => { + Err("supervisor_topology=workspace is not implemented".into()) + } + Some(SupervisorTopology::Gateway) => { + let shared = self.shared_supervisor.as_ref().ok_or_else(|| { + "supervisor_topology=gateway requires shared_supervisor".to_string() + })?; + shared.validate()?; + if self.provider_spiffe_enabled() { + return Err("supervisor_topology=gateway does not support provider_spiffe_workload_api_socket_path; a shared supervisor has no per-sandbox workload identity".into()); + } + if self.https_proxy.is_some() + || self.proxy_auth_secret_name.is_some() + || self.proxy_ca_bundle.is_some() + { + return Err( + "supervisor_topology=gateway does not support upstream proxy settings" + .into(), + ); + } + Ok(()) + } + } + } + pub fn validate_allowed_pod_annotations(&self) -> Result<(), String> { validate_pod_metadata_allowlist("allowed_pod_annotations", &self.allowed_pod_annotations) } @@ -1010,6 +1113,69 @@ mod tests { use super::*; use std::collections::BTreeMap as HashMap; + #[test] + fn gateway_supervisor_topology_requires_its_connection_and_refuses_unsupported_inputs() { + let shared = || SharedSupervisorConfig { + service_name: "openshell-supervisor".to_string(), + service_namespace: "openshell".to_string(), + runtime_port: 8443, + token_secret_name: "openshell-supervisor-token".to_string(), + }; + let gateway = |shared: Option| KubernetesComputeConfig { + supervisor_topology: Some(SupervisorTopology::Gateway), + shared_supervisor: shared, + ..KubernetesComputeConfig::default() + }; + assert!( + gateway(Some(shared())) + .validate_supervisor_topology() + .is_ok() + ); + assert!(gateway(None).validate_supervisor_topology().is_err()); + // The connection alone does not opt a gateway in. + let unset = KubernetesComputeConfig { + shared_supervisor: Some(shared()), + ..KubernetesComputeConfig::default() + }; + assert!(unset.validate_supervisor_topology().is_err()); + for invalid in [ + SharedSupervisorConfig { + runtime_port: 0, + ..shared() + }, + SharedSupervisorConfig { + service_namespace: "Not_A_Label".to_string(), + ..shared() + }, + SharedSupervisorConfig { + token_secret_name: "Not_A_Secret_Name".to_string(), + ..shared() + }, + ] { + assert!( + gateway(Some(invalid)) + .validate_supervisor_topology() + .is_err() + ); + } + // A shared supervisor holds no per-sandbox workload identity or proxy credential. + let with_proxy = KubernetesComputeConfig { + https_proxy: Some("http://proxy.example:3128".to_string()), + ..gateway(Some(shared())) + }; + assert!(with_proxy.validate_supervisor_topology().is_err()); + let with_proxy_ca = KubernetesComputeConfig { + proxy_ca_bundle: Some("/etc/proxy/ca.pem".to_string()), + ..gateway(Some(shared())) + }; + assert!(with_proxy_ca.validate_supervisor_topology().is_err()); + let workspace = KubernetesComputeConfig { + supervisor_topology: Some(SupervisorTopology::Workspace), + ..KubernetesComputeConfig::default() + }; + assert!(workspace.validate_supervisor_topology().is_err()); + } + #[test] fn image_pull_policy_accepts_config_and_kubernetes_spellings() { for (value, expected) in [ diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 0a1acc7be2..91a1ff416b 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -11,16 +11,20 @@ use crate::config::{ }; use crate::isolation::{ BOUNDARY_PAIR_LABEL, BOUNDARY_ROLE_LABEL, KubernetesControlPath, - KubernetesSandboxRuntimeBoundarySpec, + KubernetesSandboxRuntimeBoundarySpec, KubernetesSupervisorClaim, }; use crate::lifecycle::LifecycleGates; use crate::sandbox_runtime::{ BOUNDARY_CERTIFICATE_PATH, BOUNDARY_CONFIG_PATH, BOUNDARY_PRIVATE_KEY_PATH, ClientTlsMaterial, - SUPERVISOR_TERMINATION_GRACE_PERIOD_SECONDS, SandboxRuntimeNames, SupervisorClientTls, - boundary_service, boundary_service_matches_mode, control_policy, generate_proxy_ca_material, - image_pull_secret_copy, sandbox_bootstrap_secret, - sandbox_owner_reference as sandbox_runtime_sandbox_owner_reference, - supervisor_bootstrap_secret, supervisor_pod, workload_fence, + SHARED_INSTANCE_KEY, SUPERVISOR_TERMINATION_GRACE_PERIOD_SECONDS, SandboxRuntimeNames, + SupervisorClientTls, boundary_service, boundary_service_matches_mode, control_policy, + generate_proxy_ca_material, image_pull_secret_copy, sandbox_bootstrap_secret, + sandbox_owner_reference as sandbox_runtime_sandbox_owner_reference, shared_control_policy, + shared_instance_secret, supervisor_bootstrap_secret, supervisor_pod, workload_fence, +}; +use crate::shared_supervisor::{ + ANNOTATION_SUPERVISOR_SESSION, ANNOTATION_SUPERVISOR_TOPOLOGY, GATEWAY_TOPOLOGY, + SharedSupervisor, SupervisorPlacement, supervisor_placement, }; use futures::{Stream, StreamExt, TryStreamExt}; use k8s_openapi::api::authentication::v1::{ @@ -70,6 +74,10 @@ use openshell_sandbox_backend::boundary_protocol::{ GatewayVerificationKey, SandboxTlsClientConfig, SandboxTlsServerConfig, generate_sandbox_tls_material, }; +use openshell_supervisor_control::{ + InstanceSupervision, LaunchError, RecoverableInstance, RecoveryBaseline, + RegistrationTraceContext, SandboxInstanceRegistration, SharedSupervision, +}; use serde::Deserialize; use std::collections::{BTreeMap, HashSet}; use std::path::{Path, PathBuf}; @@ -188,6 +196,64 @@ struct StagedBootstrapMaterial { client_tls: Option, } +/// The supervisor a generation is installed with. +enum GenerationSupervisor { + /// A supervisor Pod paired with this generation's workload Pod. + Dedicated { + pod_uid: String, + connection: KubernetesControlConnection, + }, + /// A sandbox instance on the gateway's shared supervisor. + Shared { route: String }, +} + +impl GenerationSupervisor { + /// Third component of the runtime identity. An instance route can never + /// equal a Pod UID, so no Pod can authenticate as a shared generation. + fn runtime_identity_component(&self) -> &str { + match self { + Self::Dedicated { pod_uid, .. } => pod_uid, + Self::Shared { route } => route, + } + } +} + +/// A generation's supervisor with everything installation binds into its +/// boundary evidence, resolved and verified once. +enum GenerationLaunchTarget { + Dedicated { + pod_uid: String, + /// Cluster IP of the boundary Service of the recorded connection mode. + service_ip: std::net::IpAddr, + /// Present exactly when the runtime initiates the control connection. + control_policy: Option, + }, + Shared { + shared: SharedSupervisor, + route: String, + target: SharedSupervisorTarget, + control_policy: VerifiedControlPolicy, + }, +} + +impl GenerationLaunchTarget { + fn control_policy(&self) -> Option<&VerifiedControlPolicy> { + match self { + Self::Dedicated { control_policy, .. } => control_policy.as_ref(), + Self::Shared { control_policy, .. } => Some(control_policy), + } + } +} + +/// Where workload runtimes reach the gateway's shared supervisor. +struct SharedSupervisorTarget { + namespace: String, + authority: String, + address: std::net::SocketAddr, + /// Labels of the shared supervisor Pods, from their Service. + selector: BTreeMap, +} + /// Stored coordinates of a validated runtime-initiated control policy. struct VerifiedControlPolicy { uid: String, @@ -353,6 +419,7 @@ fn admission_error(error: tonic::Status) -> KubernetesDriverError { } } const SANDBOX_RUNTIME_RECONCILE_INTERVAL: Duration = Duration::from_secs(30); +/// Unsupervised shared sandbox instances removed at once by one reconcile pass. /// Bound how long a crash-interrupted, fail-closed bootstrap may remain stranded. const SANDBOX_RUNTIME_BOOTSTRAP_GRACE: Duration = Duration::from_mins(5); @@ -828,6 +895,9 @@ pub struct KubernetesComputeDriver { lifecycle_gates: Arc, config: KubernetesComputeConfig, operator_allowlist: Option, + /// Desired shared sandbox instances while new generations use the gateway + /// topology. + shared_supervisor: Option, } impl std::fmt::Debug for KubernetesComputeDriver { @@ -876,12 +946,51 @@ impl KubernetesComputeDriver { lifecycle_gates: Arc::default(), config, operator_allowlist: None, + shared_supervisor: None, } } + /// A driver whose new generations get a dedicated supervisor Pod each. + /// A configuration that places them on the gateway's shared supervisor + /// needs [`Self::with_shared_supervision`]; without the gateway's + /// coordination it is refused here, before anything runs. pub async fn new( config: KubernetesComputeConfig, shutdown_rx: tokio::sync::watch::Receiver, + ) -> Result { + if config.shared_supervisor.is_some() { + return Err(KubernetesDriverError::Precondition( + "the gateway supervisor topology requires the gateway's shared supervision, which only the driver linked into the gateway receives" + .to_string(), + )); + } + Self::build(config, shutdown_rx, None).await + } + + /// A driver whose new generations become sandbox instances on the shared + /// supervisor the gateway coordinates through `supervision`. The driver + /// adapts that coordination to Kubernetes: it records placements, + /// persists and reads back launch material, provisions egress to the + /// supervisor's Service, and delivers the supervisor token into its + /// Secret. + pub async fn with_shared_supervision( + config: KubernetesComputeConfig, + shutdown_rx: tokio::sync::watch::Receiver, + supervision: SharedSupervision, + ) -> Result { + if config.shared_supervisor.is_none() { + return Err(KubernetesDriverError::Precondition( + "shared supervision was provided but the configuration does not place generations on the shared supervisor" + .to_string(), + )); + } + Self::build(config, shutdown_rx, Some(supervision)).await + } + + async fn build( + config: KubernetesComputeConfig, + shutdown_rx: tokio::sync::watch::Receiver, + supervision: Option, ) -> Result { config .resource_admission @@ -908,6 +1017,9 @@ impl KubernetesComputeDriver { config .validate_allowed_pod_labels() .map_err(KubernetesDriverError::Precondition)?; + config + .validate_supervisor_topology() + .map_err(KubernetesDriverError::Precondition)?; // Read the bundle once at startup so an unreadable path or a // certificate-free file fails the gateway here, rather than as a // repeated sandbox-create failure. The authoritative read still @@ -956,6 +1068,22 @@ impl KubernetesComputeDriver { None }; + let shared_supervisor = config + .shared_supervisor + .clone() + .zip(supervision) + .map(|(config, supervision)| SharedSupervisor::new(config, supervision)); + if let Some(shared) = &shared_supervisor { + shared.spawn_token_secret_writer(client.clone(), shutdown_rx.clone()); + } + if shared_supervisor.is_some() + && (config.supervisor_otlp_endpoint.is_some() + || config.supervisor_ocsf_export.is_some()) + { + warn!( + "the gateway supervisor topology does not export supervisor OTLP traces or OCSF events; shared sandbox instances keep console logs and gateway log push" + ); + } let driver = Self { client, watch_client, @@ -963,6 +1091,7 @@ impl KubernetesComputeDriver { lifecycle_gates: Arc::default(), config, operator_allowlist, + shared_supervisor, }; if driver.workspace_mode() == WorkspaceMode::Shared { @@ -1920,12 +2049,15 @@ impl KubernetesComputeDriver { .namespace .clone() .unwrap_or_else(|| self.config.namespace.clone()); - Ok( - sandbox_from_object_with_sandbox_runtime_readiness(&self.client, &ns, obj) - .await - .ok() - .map(|(_, sandbox)| sandbox), + Ok(sandbox_from_object_with_sandbox_runtime_readiness( + &self.client, + self.shared_supervisor.as_ref(), + &ns, + obj, ) + .await + .ok() + .map(|(_, sandbox)| sandbox)) } Ok(Err(err)) => { warn!( @@ -1976,8 +2108,13 @@ impl KubernetesComputeDriver { .namespace .clone() .unwrap_or_else(|| self.config.namespace.clone()); - match sandbox_from_object_with_sandbox_runtime_readiness(&self.client, &ns, obj) - .await + match sandbox_from_object_with_sandbox_runtime_readiness( + &self.client, + self.shared_supervisor.as_ref(), + &ns, + obj, + ) + .await { Ok((_, sandbox)) => sandboxes.push(sandbox), Err(err) => { @@ -2105,6 +2242,7 @@ impl KubernetesComputeDriver { .await; let generation = random_sandbox_runtime_token()?; + let placement = self.new_generation_placement(); let proxy_names = SandboxRuntimeNames::for_generation(&sandbox.id, &generation); let image_pull_secrets = self.generation_image_pull_secret_names(&proxy_names); let main_process_spec = openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec( @@ -2199,6 +2337,13 @@ impl KubernetesComputeDriver { ANNOTATION_SANDBOX_RUNTIME_CONTROL_GENERATION.to_string(), generation.clone(), ); + if let SupervisorPlacement::Shared { route } = &placement { + annotations.insert( + ANNOTATION_SUPERVISOR_TOPOLOGY.to_string(), + GATEWAY_TOPOLOGY.to_string(), + ); + annotations.insert(ANNOTATION_SUPERVISOR_SESSION.to_string(), route.clone()); + } for key in [ crate::config::ANNOTATION_SCC_UID_RANGE, crate::config::ANNOTATION_SCC_SUPPLEMENTAL_GROUPS, @@ -2261,6 +2406,7 @@ impl KubernetesComputeDriver { &created, &proxy_names, control_connection, + &placement, resolved_user_id, resolved_group_id, &main_process_spec, @@ -2271,6 +2417,7 @@ impl KubernetesComputeDriver { Ok(runtime_identity) => runtime_identity, Err(error) => { warn!(sandbox_id = %sandbox.id, %error, "sandbox-runtime provisioning failed; rolling back Sandbox CR"); + self.remove_shared_instance(&placement); let _ = agent_sandbox_api .api .delete(&kube_name, &DeleteParams::default()) @@ -2317,7 +2464,7 @@ impl KubernetesComputeDriver { namespace: &str, names: &SandboxRuntimeNames, service_ip: std::net::IpAddr, - control_policy: Option, + control_policy: Option<&VerifiedControlPolicy>, ) -> KubernetesControlPath { let port = self.config.sandbox_runtime.boundary_port; let unspecified = std::net::SocketAddr::new( @@ -2345,8 +2492,8 @@ impl KubernetesComputeDriver { control_authority, supervisor_address: service_address, supervisor_listen: unspecified, - control_policy_uid: policy.uid, - control_policy_resource_version: policy.resource_version, + control_policy_uid: policy.uid.clone(), + control_policy_resource_version: policy.resource_version.clone(), }, } } @@ -2364,6 +2511,11 @@ impl KubernetesComputeDriver { ) -> Result<(), KubernetesDriverError> { let mode = sandbox_runtime_control_connection(object)?; let names = SandboxRuntimeNames::new(sandbox_id); + if let SupervisorPlacement::Shared { .. } = supervisor_placement(object)? { + return self + .verify_sandbox_runtime_shared_control_policy(object, namespace, &names) + .await; + } let service_name = names.control_service(mode); let services = Api::::namespaced(self.client.clone(), namespace); let service = tokio::time::timeout(KUBE_API_TIMEOUT, services.get_opt(service_name)) @@ -2431,6 +2583,53 @@ impl KubernetesComputeDriver { Ok(()) } + /// A shared generation's egress allowance must remain the object bound + /// into its boundary evidence; any change can widen workload egress. + async fn verify_sandbox_runtime_shared_control_policy( + &self, + object: &DynamicObject, + namespace: &str, + names: &SandboxRuntimeNames, + ) -> Result<(), KubernetesDriverError> { + let policies = Api::::namespaced(self.client.clone(), namespace); + let stored = tokio::time::timeout( + KUBE_API_TIMEOUT, + policies.get_opt(&names.shared_control_policy), + ) + .await + .map_err(|_| { + KubernetesDriverError::Message( + "timed out reading the shared control NetworkPolicy".to_string(), + ) + })? + .map_err(KubernetesDriverError::from_kube)?; + // A bootstrap still installing has not bound its policy yet. + if stored.is_none() + && sandbox_runtime_bootstrap_phase(object) + == Some(SandboxRuntimeBootstrapPhase::Preparing) + { + return Ok(()); + } + let stored = stored.ok_or_else(|| { + KubernetesDriverError::Precondition(format!( + "shared control NetworkPolicy {} is missing", + names.shared_control_policy + )) + })?; + if !sandbox_runtime_policy_generation_matches( + &stored, + object, + ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_UID, + ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_GENERATION, + ) { + return Err(KubernetesDriverError::Precondition(format!( + "shared control NetworkPolicy {} changed after it was bound", + names.shared_control_policy + ))); + } + Ok(()) + } + async fn create_sandbox_runtime_fence( &self, namespace: &str, @@ -2711,6 +2910,7 @@ impl KubernetesComputeDriver { sandbox_cr: &DynamicObject, names: &SandboxRuntimeNames, control_connection: KubernetesControlConnection, + placement: &SupervisorPlacement, agent_uid: u32, agent_gid: u32, main_process_spec: &str, @@ -2736,35 +2936,48 @@ impl KubernetesComputeDriver { &sandbox_api.resource.api_version, false, ); - // Preserve the legacy workload endpoint even for newly created - // sandboxes: an old gateway may handle their next start mid-rollout. - for mode in [ - KubernetesControlConnection::SupervisorInitiated, - KubernetesControlConnection::RuntimeInitiated, - ] { - self.ensure_sandbox_runtime_service( - namespace, - names, - &sandbox.id, - mode, - dependent_owner.clone(), - ) - .await?; - } - let supervisor_uid = self - .create_sandbox_runtime_supervisor( - namespace, - names, - &sandbox.id, - &sandbox.name, - dependent_owner, - agent_uid, - agent_gid, - main_process_spec, - log_level, - staged.upstream_proxy_ca_bundle.is_some(), - ) - .await?; + let supervisor = match placement { + SupervisorPlacement::Dedicated => { + // Preserve the legacy workload endpoint even for newly created + // sandboxes: an old gateway may handle their next start mid-rollout. + for mode in [ + KubernetesControlConnection::SupervisorInitiated, + KubernetesControlConnection::RuntimeInitiated, + ] { + self.ensure_sandbox_runtime_service( + namespace, + names, + &sandbox.id, + mode, + dependent_owner.clone(), + ) + .await?; + } + let pod_uid = self + .create_sandbox_runtime_supervisor( + namespace, + names, + &sandbox.id, + &sandbox.name, + dependent_owner, + agent_uid, + agent_gid, + main_process_spec, + log_level, + staged.upstream_proxy_ca_bundle.is_some(), + ) + .await?; + GenerationSupervisor::Dedicated { + pod_uid, + connection: control_connection, + } + } + // The shared supervisor already runs; this generation becomes a + // sandbox instance on it during installation. + SupervisorPlacement::Shared { route } => GenerationSupervisor::Shared { + route: route.clone(), + }, + }; // Release the Sandbox CR only far enough for the controller to create // the workload Pod. The Pod remains unschedulable because its template @@ -2793,8 +3006,7 @@ impl KubernetesComputeDriver { &sandbox.id, cr_uid, names, - control_connection, - &supervisor_uid, + &supervisor, agent_uid, agent_gid, child_env, @@ -2807,11 +3019,11 @@ impl KubernetesComputeDriver { // control process without deadlocking behind this driver call. Only // boundary PID 1 is running at this point; the agent process cannot // start until control attaches and confirms enforcement. Reconcile - // removes the marker after the supervisor Pod becomes Ready. + // removes the marker after the supervisor becomes ready. Ok(kubernetes_runtime_identity( namespace, cr_uid, - &supervisor_uid, + supervisor.runtime_identity_component(), )) } @@ -2851,6 +3063,34 @@ impl KubernetesComputeDriver { Ok(()) } + /// Placement for a generation this driver starts now. + fn new_generation_placement(&self) -> SupervisorPlacement { + if self.shared_supervisor.is_some() { + SupervisorPlacement::Shared { + route: openshell_sandbox_backend::runtime_routed::new_route_handle(), + } + } else { + SupervisorPlacement::Dedicated + } + } + + /// Withdraw a shared generation's sandbox instance, if any, so the host + /// stops it. Instance loss also ends supervision when the workload is + /// suspended, so a missing configuration only logs. + fn remove_shared_instance(&self, placement: &SupervisorPlacement) { + let SupervisorPlacement::Shared { route } = placement else { + return; + }; + if let Some(shared) = &self.shared_supervisor { + shared.supervision().withdraw(route); + } else { + warn!( + route, + "shared supervisor is not configured; its sandbox instance cannot be withdrawn" + ); + } + } + /// Read operator-provided bootstrap material once per generation, so the /// whole generation stages, mounts, and references the same bytes, and so /// unreadable material fails before any Service, Pod, or Secret is created. @@ -2927,6 +3167,179 @@ impl KubernetesComputeDriver { .ok_or_else(|| KubernetesDriverError::Message("supervisor Pod has no UID".to_string())) } + /// Read the gateway's shared supervisor Service: its address and the + /// labels of the Pods it selects. + async fn shared_supervisor_target( + &self, + ) -> Result<(SharedSupervisor, SharedSupervisorTarget), KubernetesDriverError> { + let shared = self.shared_supervisor.as_ref().ok_or_else(|| { + KubernetesDriverError::Precondition( + "the gateway supervisor topology is not configured".to_string(), + ) + })?; + let config = shared.config(); + let services = Api::::namespaced(self.client.clone(), &config.service_namespace); + let service = tokio::time::timeout(KUBE_API_TIMEOUT, services.get(&config.service_name)) + .await + .map_err(|_| { + KubernetesDriverError::Message( + "timed out reading the shared supervisor Service".to_string(), + ) + })? + .map_err(KubernetesDriverError::from_kube)?; + let spec = service.spec.unwrap_or_default(); + let selector = spec + .selector + .filter(|selector| !selector.is_empty()) + .ok_or_else(|| { + KubernetesDriverError::Precondition( + "the shared supervisor Service selects no Pods".to_string(), + ) + })?; + let address: std::net::IpAddr = spec + .cluster_ip + .filter(|ip| ip != "None") + .ok_or_else(|| { + KubernetesDriverError::Precondition( + "the shared supervisor Service has no ClusterIP".to_string(), + ) + })? + .parse() + .map_err(|error| { + KubernetesDriverError::Message(format!( + "invalid shared supervisor Service ClusterIP: {error}" + )) + })?; + Ok(( + shared.clone(), + SharedSupervisorTarget { + namespace: config.service_namespace.clone(), + authority: format!("{}:{}", config.service_host(), config.runtime_port), + address: std::net::SocketAddr::new(address, config.runtime_port), + selector, + }, + )) + } + + /// Create or validate this sandbox's egress allowance to the shared + /// supervisor and return the stored coordinates bound into its evidence. + async fn ensure_sandbox_runtime_shared_control_policy( + &self, + namespace: &str, + names: &SandboxRuntimeNames, + sandbox_id: &str, + target: &SharedSupervisorTarget, + owner: OwnerReference, + ) -> Result { + let policies: Api = Api::namespaced(self.client.clone(), namespace); + let expected = shared_control_policy( + namespace, + names, + sandbox_id, + &target.namespace, + target.selector.clone(), + target.address.port(), + owner, + ); + create_or_validate_sandbox_runtime_fence(&policies, &expected).await?; + let stored = + tokio::time::timeout(KUBE_API_TIMEOUT, policies.get(&names.shared_control_policy)) + .await + .map_err(|_| { + KubernetesDriverError::Message( + "timed out reading the shared control NetworkPolicy".to_string(), + ) + })? + .map_err(KubernetesDriverError::from_kube)?; + validate_sandbox_runtime_fence(&stored, &expected)?; + VerifiedControlPolicy::from_policy(&stored) + } + + /// Resolve and verify where `supervisor` serves this generation: the + /// boundary Service and, for runtime-initiated connections, the control + /// policy of a dedicated supervisor, or the shared supervisor's Service and + /// this sandbox's egress allowance to it. + async fn resolve_generation_launch_target( + &self, + namespace: &str, + names: &SandboxRuntimeNames, + sandbox_id: &str, + supervisor: &GenerationSupervisor, + policy_owner: OwnerReference, + ) -> Result { + match supervisor { + GenerationSupervisor::Dedicated { + pod_uid, + connection, + } => { + let services = Api::::namespaced(self.client.clone(), namespace); + let service_name = names.control_service(*connection); + let service = services + .get(service_name) + .await + .map_err(KubernetesDriverError::from_kube)?; + // Running generations must retain the Service role prepared for them. + if !boundary_service_matches_mode(&service, sandbox_id, *connection) { + return Err(KubernetesDriverError::Precondition(format!( + "boundary Service {} does not select the {} listener recorded for this sandbox", + service_name, + connection.as_str() + ))); + } + let service_ip: std::net::IpAddr = service + .spec + .and_then(|spec| spec.cluster_ip) + .filter(|ip| ip != "None") + .ok_or_else(|| { + KubernetesDriverError::Message( + "boundary Service has no ClusterIP".to_string(), + ) + })? + .parse() + .map_err(|error| { + KubernetesDriverError::Message(format!( + "invalid boundary Service ClusterIP: {error}" + )) + })?; + let control_policy = match connection { + KubernetesControlConnection::SupervisorInitiated => None, + KubernetesControlConnection::RuntimeInitiated => Some( + self.ensure_sandbox_runtime_control_policy( + namespace, + names, + sandbox_id, + policy_owner, + ) + .await?, + ), + }; + Ok(GenerationLaunchTarget::Dedicated { + pod_uid: pod_uid.clone(), + service_ip, + control_policy, + }) + } + GenerationSupervisor::Shared { route } => { + let (shared, target) = self.shared_supervisor_target().await?; + let control_policy = self + .ensure_sandbox_runtime_shared_control_policy( + namespace, + names, + sandbox_id, + &target, + policy_owner, + ) + .await?; + Ok(GenerationLaunchTarget::Shared { + shared, + route: route.clone(), + target, + control_policy, + }) + } + } + } + #[allow(clippy::too_many_arguments, clippy::similar_names)] async fn install_sandbox_runtime_generation( &self, @@ -2936,8 +3349,7 @@ impl KubernetesComputeDriver { sandbox_id: &str, cr_uid: &str, names: &SandboxRuntimeNames, - control_connection: KubernetesControlConnection, - supervisor_uid: &str, + supervisor: &GenerationSupervisor, agent_uid: u32, agent_gid: u32, child_env: std::collections::HashMap, @@ -2953,33 +3365,6 @@ impl KubernetesComputeDriver { .ok_or_else(|| { KubernetesDriverError::Message("sandbox namespace has no UID".to_string()) })?; - let services = Api::::namespaced(self.client.clone(), namespace); - let service_name = names.control_service(control_connection); - let service = services - .get(service_name) - .await - .map_err(KubernetesDriverError::from_kube)?; - // Running generations must retain the Service role prepared for them. - if !boundary_service_matches_mode(&service, sandbox_id, control_connection) { - return Err(KubernetesDriverError::Precondition(format!( - "boundary Service {} does not select the {} listener recorded for this sandbox", - service_name, - control_connection.as_str() - ))); - } - let service_ip: std::net::IpAddr = service - .spec - .and_then(|spec| spec.cluster_ip) - .filter(|ip| ip != "None") - .ok_or_else(|| { - KubernetesDriverError::Message("boundary Service has no ClusterIP".to_string()) - })? - .parse() - .map_err(|error| { - KubernetesDriverError::Message(format!( - "invalid boundary Service ClusterIP: {error}" - )) - })?; let policies = Api::::namespaced(self.client.clone(), namespace); let fence = policies .get(&names.workload_policy) @@ -2996,23 +3381,20 @@ impl KubernetesComputeDriver { "workload NetworkPolicy has no resourceVersion".to_string(), ) })?; - let control_policy = match control_connection { - KubernetesControlConnection::SupervisorInitiated => None, - KubernetesControlConnection::RuntimeInitiated => Some( - self.ensure_sandbox_runtime_control_policy( - namespace, - names, - sandbox_id, - sandbox_runtime_sandbox_owner_reference( - cr_name, - cr_uid, - &sandbox_api.resource.api_version, - false, - ), - ) - .await?, - ), - }; + let target = self + .resolve_generation_launch_target( + namespace, + names, + sandbox_id, + supervisor, + sandbox_runtime_sandbox_owner_reference( + cr_name, + cr_uid, + &sandbox_api.resource.api_version, + false, + ), + ) + .await?; let pods = Api::::namespaced(self.client.clone(), namespace); let workload_pod = self @@ -3043,11 +3425,13 @@ impl KubernetesComputeDriver { })?; let mut annotations = serde_json::json!({ ANNOTATION_SANDBOX_RUNTIME_WORKLOAD_UID: workload_pod_uid.clone(), - ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID: supervisor_uid, ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_UID: fence_uid.clone(), ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_GENERATION: fence_generation.to_string(), }); - if let Some(policy) = &control_policy { + if let GenerationLaunchTarget::Dedicated { pod_uid, .. } = &target { + annotations[ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID] = pod_uid.clone().into(); + } + if let Some(policy) = target.control_policy() { annotations[ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_UID] = policy.uid.clone().into(); annotations[ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_GENERATION] = policy.generation.to_string().into(); @@ -3073,6 +3457,37 @@ impl KubernetesComputeDriver { format!("sandbox:{cr_uid}"), ) .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; + let (supervisor_claim, control_path) = match &target { + GenerationLaunchTarget::Dedicated { + pod_uid, + service_ip, + control_policy, + } => ( + KubernetesSupervisorClaim::Pod(pod_uid.clone()), + self.sandbox_runtime_control_path( + namespace, + names, + *service_ip, + control_policy.as_ref(), + ), + ), + GenerationLaunchTarget::Shared { + route, + target, + control_policy, + .. + } => ( + KubernetesSupervisorClaim::SandboxInstance(route.clone()), + KubernetesControlPath::RuntimeInitiatedShared { + control_authority: target.authority.clone(), + supervisor_address: target.address, + supervisor_namespace: target.namespace.clone(), + route: route.clone(), + control_policy_uid: control_policy.uid.clone(), + control_policy_resource_version: control_policy.resource_version.clone(), + }, + ), + }; let provisioned = KubernetesSandboxRuntimeBoundarySpec { boundary_id: sandbox_id.to_string(), generation: launch_authentication @@ -3088,15 +3503,10 @@ impl KubernetesComputeDriver { sandbox_resource_uid: cr_uid.to_string(), workload_pod_uid: workload_pod_uid.clone(), workload_pod_uid_path: PathBuf::from(SANDBOX_POD_UID_PATH), - supervisor_pod_uid: supervisor_uid.to_string(), + supervisor: supervisor_claim, egress_policy_uid: fence_uid.clone(), egress_policy_resource_version: fence_resource_version.clone(), - control_path: self.sandbox_runtime_control_path( - namespace, - names, - service_ip, - control_policy, - ), + control_path, sandbox_tls: SandboxTlsServerConfig { certificate_chain_path: PathBuf::from(BOUNDARY_CERTIFICATE_PATH), private_key_path: PathBuf::from(BOUNDARY_PRIVATE_KEY_PATH), @@ -3111,12 +3521,8 @@ impl KubernetesComputeDriver { } .provision() .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; - let descriptor = provisioned - .runtime_descriptor - .backend_descriptor() - .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; let workload_owner = runtime_pod_owner(&workload_pod_name, &workload_pod_uid); - let supervisor_owner = runtime_pod_owner(&names.supervisor_pod, supervisor_uid); + let instance_owner = workload_owner.clone(); let sandbox_secret = sandbox_bootstrap_secret( namespace, names, @@ -3129,44 +3535,106 @@ impl KubernetesComputeDriver { tls.private_key_pem.into_bytes(), workload_owner.clone(), ); - let supervisor_secret = supervisor_bootstrap_secret( - namespace, - names, - sandbox_id, - descriptor.payload, - serde_json::to_vec(&launch_authentication.supervisor).map_err(|error| { - KubernetesDriverError::Message(format!("encode supervisor auth bundle: {error}")) - })?, - proxy_ca.certificate_pem.into_bytes(), - proxy_ca.private_key_pem.into_bytes(), - staged.upstream_proxy_ca_bundle, - staged.client_tls, - supervisor_owner.clone(), - ); let secrets = Api::::namespaced(self.client.clone(), namespace); secrets .create(&PostParams::default(), &sandbox_secret) .await .map_err(KubernetesDriverError::from_kube)?; - secrets - .create(&PostParams::default(), &supervisor_secret) - .await - .map_err(KubernetesDriverError::from_kube)?; - self.create_generation_image_pull_secrets( - namespace, - names, - sandbox_id, - vec![workload_owner, supervisor_owner], - ) - .await?; - - pods.patch( - &names.supervisor_pod, - &PatchParams::default(), - &Patch::Merge(&serde_json::json!({"spec": {"schedulingGates": []}})), - ) - .await - .map_err(KubernetesDriverError::from_kube)?; + match &target { + GenerationLaunchTarget::Dedicated { pod_uid, .. } => { + let descriptor = provisioned + .runtime_descriptor + .backend_descriptor() + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; + let supervisor_owner = runtime_pod_owner(&names.supervisor_pod, pod_uid); + let supervisor_secret = supervisor_bootstrap_secret( + namespace, + names, + sandbox_id, + descriptor.payload, + serde_json::to_vec(&launch_authentication.supervisor).map_err(|error| { + KubernetesDriverError::Message(format!( + "encode supervisor auth bundle: {error}" + )) + })?, + proxy_ca.certificate_pem.into_bytes(), + proxy_ca.private_key_pem.into_bytes(), + staged.upstream_proxy_ca_bundle, + staged.client_tls, + supervisor_owner.clone(), + ); + secrets + .create(&PostParams::default(), &supervisor_secret) + .await + .map_err(KubernetesDriverError::from_kube)?; + self.create_generation_image_pull_secrets( + namespace, + names, + sandbox_id, + vec![workload_owner, supervisor_owner], + ) + .await?; + pods.patch( + &names.supervisor_pod, + &PatchParams::default(), + &Patch::Merge(&serde_json::json!({"spec": {"schedulingGates": []}})), + ) + .await + .map_err(KubernetesDriverError::from_kube)?; + } + GenerationLaunchTarget::Shared { shared, route, .. } => { + self.create_generation_image_pull_secrets( + namespace, + names, + sandbox_id, + vec![workload_owner], + ) + .await?; + let registration = shared_instance_registration( + &admission_object, + route, + sandbox_id, + provisioned.runtime_descriptor, + launch_authentication, + proxy_ca, + )?; + // The coordinator persists the launch material with the + // generation before it assigns the instance, so a restarted + // gateway rebuilds the host's assignments from the + // generation's own resources; assignment then publishes it to + // the host without waiting for the runtime. + shared + .supervision() + .launch(®istration, |launch_material| async { + secrets + .create( + &PostParams::default(), + &shared_instance_secret( + namespace, + names, + sandbox_id, + launch_material, + instance_owner, + ), + ) + .await + .map(|_| ()) + .map_err(KubernetesDriverError::from_kube) + }) + .await + .map_err(|error| match error { + LaunchError::Persist(error) => error, + // The host is at capacity; the create fails before + // its workload runs. + LaunchError::Refused(error) => { + KubernetesDriverError::Precondition(error.to_string()) + } + LaunchError::Invalid(error) => { + KubernetesDriverError::Message(error.to_string()) + } + })?; + } + } pods.patch( &workload_pod_name, &PatchParams::default(), @@ -3178,10 +3646,22 @@ impl KubernetesComputeDriver { sandbox_runtime_bootstrap_phase_patch(version, SandboxRuntimeBootstrapPhase::Released) }) .await?; + let control = match target { + GenerationLaunchTarget::Dedicated { .. } => BootstrapControl::SupervisorPod { + pods, + name: names.supervisor_pod.clone(), + }, + GenerationLaunchTarget::Shared { shared, route, .. } => { + BootstrapControl::SharedInstance { + shared, + route, + sandbox_id: sandbox_id.to_string(), + } + } + }; spawn_sandbox_runtime_bootstrap_completion( - pods, + control, sandbox_api.api.clone(), - names.supervisor_pod.clone(), cr_name.to_string(), Some(cr_uid.to_string()), ); @@ -3205,10 +3685,10 @@ impl KubernetesComputeDriver { } async fn stop_sandbox_inner(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { - let (agent_sandbox_api, kube_name, pod_name, namespace, stop_timeout, phase) = + let (agent_sandbox_api, kube_name, pod_name, namespace, stop_timeout, phase, placement) = self.prepare_sandbox_stop(sandbox_id).await?; if phase != Some(SandboxRuntimeBootstrapPhase::Suspending) { - self.delete_sandbox_runtime_supervisor_pod(sandbox_id, &namespace) + self.release_generation_supervisor(sandbox_id, &namespace, &placement) .await?; patch_dynamic_object_with_resource_version_retry( &agent_sandbox_api.api, @@ -3360,7 +3840,14 @@ impl KubernetesComputeDriver { .namespace .as_deref() .unwrap_or(&self.config.namespace); - if sandbox_runtime_control_availability(&self.client, namespace, sandbox_id).await + if generation_control_availability( + &self.client, + self.shared_supervisor.as_ref(), + namespace, + sandbox_id, + &object, + ) + .await == SandboxRuntimeControlAvailability::Available && sandbox_runtime_bootstrap_is_ready_to_complete(&object) { @@ -3369,15 +3856,14 @@ impl KubernetesComputeDriver { let cr_uid = object.metadata.uid.as_deref().ok_or_else(|| { KubernetesDriverError::Message("sandbox resource has no UID".to_string()) })?; - let supervisor_uid = required_sandbox_annotation( - &object, - ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID, - )?; - return Ok(kubernetes_runtime_identity( - namespace, - cr_uid, - &supervisor_uid, - )); + let supervisor = match supervisor_placement(&object)? { + SupervisorPlacement::Dedicated => required_sandbox_annotation( + &object, + ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID, + )?, + SupervisorPlacement::Shared { route } => route, + }; + return Ok(kubernetes_runtime_identity(namespace, cr_uid, &supervisor)); } } @@ -3440,6 +3926,7 @@ impl KubernetesComputeDriver { } let control_connection = KubernetesControlConnection::RuntimeInitiated; + let placement = self.new_generation_placement(); let names = SandboxRuntimeNames::for_generation(sandbox_id, generation.as_str()); self.create_sandbox_runtime_fence(&namespace, &names) .await?; @@ -3448,35 +3935,55 @@ impl KubernetesComputeDriver { self.delete_sandbox_runtime_supervisor(sandbox_id, &namespace, &stale_generations) .await?; // The legacy Service keeps selecting the workload for old gateways. - // Add the supervisor endpoint when upgrading a legacy sandbox. - let services = Api::::namespaced(self.client.clone(), &namespace); - let service = services - .get(&names.boundary_service) - .await - .map_err(KubernetesDriverError::from_kube)?; - if !boundary_service_matches_mode( - &service, - sandbox_id, - KubernetesControlConnection::SupervisorInitiated, - ) { - return Err(KubernetesDriverError::Precondition(format!( - "boundary Service {} no longer selects the workload", - names.boundary_service - ))); - } - self.ensure_sandbox_runtime_service( - &namespace, - &names, - sandbox_id, - control_connection, - sandbox_runtime_sandbox_owner_reference( + // Add the supervisor endpoint when upgrading a legacy sandbox. A + // shared generation has no endpoint of its own, so a dedicated + // generation that follows one creates both. + if placement == SupervisorPlacement::Dedicated { + let owner = sandbox_runtime_sandbox_owner_reference( cr_name, cr_uid, &sandbox_api.resource.api_version, false, - ), - ) - .await?; + ); + let services = Api::::namespaced(self.client.clone(), &namespace); + let legacy = services + .get_opt(&names.boundary_service) + .await + .map_err(KubernetesDriverError::from_kube)?; + match legacy { + Some(service) + if !boundary_service_matches_mode( + &service, + sandbox_id, + KubernetesControlConnection::SupervisorInitiated, + ) => + { + return Err(KubernetesDriverError::Precondition(format!( + "boundary Service {} no longer selects the workload", + names.boundary_service + ))); + } + Some(_) => {} + None => { + self.ensure_sandbox_runtime_service( + &namespace, + &names, + sandbox_id, + KubernetesControlConnection::SupervisorInitiated, + owner.clone(), + ) + .await?; + } + } + self.ensure_sandbox_runtime_service( + &namespace, + &names, + sandbox_id, + control_connection, + owner, + ) + .await?; + } let main_process_spec = required_sandbox_annotation(&object, ANNOTATION_SANDBOX_RUNTIME_MAIN_PROCESS_SPEC)?; let log_level = required_sandbox_annotation(&object, ANNOTATION_SANDBOX_RUNTIME_LOG_LEVEL)?; @@ -3502,25 +4009,33 @@ impl KubernetesComputeDriver { agent_gid, ), )?; - let supervisor_uid = self - .create_sandbox_runtime_supervisor( - &namespace, - &names, - sandbox_id, - &sandbox_name, - sandbox_runtime_sandbox_owner_reference( - cr_name, - cr_uid, - &sandbox_api.resource.api_version, - false, - ), - agent_uid, - agent_gid, - &main_process_spec, - &log_level, - staged.upstream_proxy_ca_bundle.is_some(), - ) - .await?; + let supervisor = match &placement { + SupervisorPlacement::Dedicated => GenerationSupervisor::Dedicated { + pod_uid: self + .create_sandbox_runtime_supervisor( + &namespace, + &names, + sandbox_id, + &sandbox_name, + sandbox_runtime_sandbox_owner_reference( + cr_name, + cr_uid, + &sandbox_api.resource.api_version, + false, + ), + agent_uid, + agent_gid, + &main_process_spec, + &log_level, + staged.upstream_proxy_ca_bundle.is_some(), + ) + .await?, + connection: control_connection, + }, + SupervisorPlacement::Shared { route } => GenerationSupervisor::Shared { + route: route.clone(), + }, + }; let mut volumes = object .data @@ -3553,6 +4068,16 @@ impl KubernetesComputeDriver { |version| { let mut running_patch = sandbox_operating_state_patch(&sandbox_api.resource.version, version, true); + let (supervisor_uid, topology, instance_route) = match &supervisor { + GenerationSupervisor::Dedicated { pod_uid, .. } => { + (pod_uid.as_str().into(), serde_json::Value::Null, serde_json::Value::Null) + } + GenerationSupervisor::Shared { route } => ( + serde_json::Value::Null, + GATEWAY_TOPOLOGY.into(), + route.as_str().into(), + ), + }; running_patch["metadata"]["annotations"] = serde_json::json!({ ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING: "true", ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_STARTED_AT: openshell_core::time::now_ms().to_string(), @@ -3560,9 +4085,11 @@ impl KubernetesComputeDriver { ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_PHASE: SandboxRuntimeBootstrapPhase::Preparing.as_str(), ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_SESSION: launch_authentication.supervisor.session_id.to_string(), ANNOTATION_SANDBOX_RUNTIME_GENERATION: generation.as_str(), - ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID: supervisor_uid.clone(), + ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID: supervisor_uid, ANNOTATION_SANDBOX_RUNTIME_CONTROL_CONNECTION: control_connection.as_str(), ANNOTATION_SANDBOX_RUNTIME_CONTROL_GENERATION: generation.as_str(), + ANNOTATION_SUPERVISOR_TOPOLOGY: topology, + ANNOTATION_SUPERVISOR_SESSION: instance_route, }); running_patch["spec"]["podTemplate"]["spec"] = restart_pod_template_spec( &volumes, @@ -3582,8 +4109,7 @@ impl KubernetesComputeDriver { sandbox_id, cr_uid, &names, - control_connection, - &supervisor_uid, + &supervisor, agent_uid, agent_gid, child_env, @@ -3595,6 +4121,8 @@ impl KubernetesComputeDriver { .await; if let Err(error) = restart { + // The restart patch may not have recorded the sandbox instance yet. + self.remove_shared_instance(&placement); if let Err(rollback_error) = self.stop_sandbox_inner(sandbox_id).await { return Err(KubernetesDriverError::Message(format!( "restart failed: {error}; rollback remains pending: {rollback_error}" @@ -3605,7 +4133,7 @@ impl KubernetesComputeDriver { Ok(kubernetes_runtime_identity( &namespace, cr_uid, - &supervisor_uid, + supervisor.runtime_identity_component(), )) } @@ -3620,6 +4148,7 @@ impl KubernetesComputeDriver { String, Duration, Option, + SupervisorPlacement, ), KubernetesDriverError, > { @@ -3648,6 +4177,7 @@ impl KubernetesComputeDriver { .next() .ok_or(KubernetesDriverError::NotFound)?; let phase = sandbox_runtime_bootstrap_phase(&object); + let placement = placement_for_release(&object, sandbox_id); let namespace = object .metadata .namespace @@ -3695,9 +4225,34 @@ impl KubernetesComputeDriver { namespace, stop_timeout, phase, + placement, )) } + /// End supervision of the current generation before its workload stops. + /// + /// A shared sandbox instance is removed best effort: suspending the + /// workload ends that instance's supervision anyway, and reconciliation + /// removes any instance left behind once the shared supervisor is + /// reachable. + async fn release_generation_supervisor( + &self, + sandbox_id: &str, + namespace: &str, + placement: &SupervisorPlacement, + ) -> Result<(), KubernetesDriverError> { + match placement { + SupervisorPlacement::Dedicated => { + self.delete_sandbox_runtime_supervisor_pod(sandbox_id, namespace) + .await + } + SupervisorPlacement::Shared { .. } => { + self.remove_shared_instance(placement); + Ok(()) + } + } + } + async fn delete_sandbox_runtime_supervisor_pod( &self, sandbox_id: &str, @@ -3815,7 +4370,7 @@ impl KubernetesComputeDriver { .await?; let selector = self.sandbox_lookup_selector(sandbox_id); let lp = ListParams::default().labels(&selector); - let (kube_name, obj_namespace, _workspace, preconditions, pod_name) = + let (kube_name, obj_namespace, _workspace, preconditions, pod_name, placement) = match tokio::time::timeout(KUBE_API_TIMEOUT, lookup_api.api.list(&lp)).await { Ok(Ok(list)) => { if let Some(obj) = list.items.into_iter().next() { @@ -3832,6 +4387,8 @@ impl KubernetesComputeDriver { .as_ref() .and_then(|l| l.get(LABEL_SANDBOX_WORKSPACE).cloned()) .unwrap_or_default(); + // An unreadable placement leaves any instance to reconciliation. + let placement = supervisor_placement(&obj).ok(); let pc = Preconditions { uid: obj.metadata.uid, resource_version: obj.metadata.resource_version, @@ -3845,7 +4402,7 @@ impl KubernetesComputeDriver { }) .cloned() .unwrap_or_else(|| name.clone()); - (name, ns, ws, pc, pod_name) + (name, ns, ws, pc, pod_name, placement) } None => return Ok(false), } @@ -3882,6 +4439,12 @@ impl KubernetesComputeDriver { match tokio::time::timeout(KUBE_API_TIMEOUT, delete_api.api.delete(&kube_name, &dp)).await { Ok(Ok(_response)) => { info!(sandbox_id = %sandbox_id, namespace = %obj_namespace, "Sandbox deleted from Kubernetes"); + // The workload is going away, so its sandbox instance cannot + // stop it gracefully; end the instance without delaying the + // delete. + if let Some(placement @ SupervisorPlacement::Shared { .. }) = placement { + self.remove_shared_instance(&placement); + } { let pod_api = Api::::namespaced(self.client.clone(), &obj_namespace); let deadline = tokio::time::Instant::now() @@ -3976,6 +4539,12 @@ impl KubernetesComputeDriver { return; } }; + // Begin the recovery before the list so a lifecycle edit that lands + // in between cannot be undone by the completion below. + let shared_recovery = self + .shared_supervisor + .as_ref() + .map(|shared| shared.supervision().begin_recovery()); let list = match tokio::time::timeout( KUBE_API_TIMEOUT, lookup_api @@ -3995,6 +4564,13 @@ impl KubernetesComputeDriver { } }; + // The shared supervisor's desired set is derived from the Sandbox + // resources that may run, so instances of deleted Sandboxes and of + // stops that happened while this gateway was down are withdrawn here. + if let Some(baseline) = shared_recovery { + self.recover_shared_instances(baseline, &list.items).await; + } + for object in list.items { let Ok(sandbox_id) = sandbox_id_from_object(&object) else { continue; @@ -4135,7 +4711,14 @@ impl KubernetesComputeDriver { } } if sandbox_runtime_bootstrap_in_progress(&object) { - if sandbox_runtime_control_availability(&self.client, namespace, &sandbox_id).await + if generation_control_availability( + &self.client, + self.shared_supervisor.as_ref(), + namespace, + &sandbox_id, + &object, + ) + .await == SandboxRuntimeControlAvailability::Available && sandbox_runtime_bootstrap_is_ready_to_complete(&object) { @@ -4203,7 +4786,7 @@ impl KubernetesComputeDriver { &sandbox_id, namespace, desired_running, - sandbox_runtime_generation(&object).as_slice(), + &object, ) .await { @@ -4217,8 +4800,14 @@ impl KubernetesComputeDriver { warn!(sandbox_id, %error, "failed to reconcile sandbox-runtime supervisor Pod"); } } - let availability = - sandbox_runtime_control_availability(&self.client, namespace, &sandbox_id).await; + let availability = generation_control_availability( + &self.client, + self.shared_supervisor.as_ref(), + namespace, + &sandbox_id, + &object, + ) + .await; if desired_running && availability == SandboxRuntimeControlAvailability::Unavailable { warn!( sandbox_id, @@ -4233,6 +4822,75 @@ impl KubernetesComputeDriver { } } + /// Complete a recovery of the shared supervisor's desired set from the + /// Sandbox resources that may run. The coordinator reuses current + /// assignments and otherwise reads each generation's launch Secret + /// through [`Self::read_shared_instance_material`]; it keeps the previous + /// set when any read fails and refuses a list that raced a lifecycle + /// edit. + async fn recover_shared_instances( + &self, + baseline: RecoveryBaseline, + sandboxes: &[DynamicObject], + ) { + let Some(shared) = &self.shared_supervisor else { + return; + }; + let candidates = runnable_shared_generations(sandboxes) + .into_iter() + .filter_map(|(object, route)| { + let sandbox_id = sandbox_id_from_object(object).ok()?; + let generation = sandbox_runtime_generation(object)?.to_string(); + let namespace = object + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + Some(RecoverableInstance { + route, + sandbox_id: sandbox_id.clone(), + record: SharedInstanceSecretRecord { + namespace, + sandbox_id, + generation, + }, + }) + }) + .collect(); + // Outcomes are logged by the coordinator; a conflict is derived again + // on the next reconciliation. + shared + .supervision() + .complete_recovery(baseline, candidates, |record| { + self.read_shared_instance_material(record.clone()) + }) + .await; + } + + /// Launch material persisted with a shared generation, or `None` when its + /// Secret is gone or belongs to a dedicated generation. The bytes are + /// decoded and bound to the generation by the coordinator. + async fn read_shared_instance_material( + &self, + record: SharedInstanceSecretRecord, + ) -> Result>, KubernetesDriverError> { + let names = SandboxRuntimeNames::for_generation(&record.sandbox_id, &record.generation); + let secrets = Api::::namespaced(self.client.clone(), &record.namespace); + let secret = + tokio::time::timeout(KUBE_API_TIMEOUT, secrets.get_opt(&names.supervisor_secret)) + .await + .map_err(|_| { + KubernetesDriverError::Message( + "timed out reading the shared sandbox instance launch Secret".to_string(), + ) + })? + .map_err(KubernetesDriverError::from_kube)?; + Ok(secret + .and_then(|secret| secret.data) + .and_then(|mut data| data.remove(SHARED_INSTANCE_KEY)) + .map(|material| material.0)) + } + async fn reconcile_sandbox_runtime_suspending( &self, lookup_api: &AgentSandboxApi, @@ -4257,6 +4915,10 @@ impl KubernetesComputeDriver { return; } } + // Completion forgets the placement, so withdraw a shared sandbox instance now. + if let Ok(placement) = supervisor_placement(object) { + self.remove_shared_instance(&placement); + } if let Err(error) = self .delete_sandbox_runtime_supervisor( sandbox_id, @@ -4302,8 +4964,9 @@ impl KubernetesComputeDriver { namespace: &str, cr_name: &str, ) { + let placement = placement_for_release(object, sandbox_id); if let Err(error) = self - .delete_sandbox_runtime_supervisor_pod(sandbox_id, namespace) + .release_generation_supervisor(sandbox_id, namespace, &placement) .await { warn!(sandbox_id, %error, "could not release sandbox-runtime control; reconciliation will retry"); @@ -4530,13 +5193,18 @@ impl KubernetesComputeDriver { sandbox_id: &str, namespace: &str, desired_running: bool, - generations: &[&str], + object: &DynamicObject, ) -> Result<(), KubernetesDriverError> { + let generations = sandbox_runtime_generation(object); if !desired_running { return self - .delete_sandbox_runtime_supervisor(sandbox_id, namespace, generations) + .delete_sandbox_runtime_supervisor(sandbox_id, namespace, generations.as_slice()) .await; } + if let SupervisorPlacement::Shared { .. } = supervisor_placement(object)? { + // Sandbox instance availability is part of control availability. + return Ok(()); + } // The boundary Service was checked by verify_sandbox_runtime_control_path. let names = SandboxRuntimeNames::new(sandbox_id); let pods = Api::::namespaced(self.client.clone(), namespace); @@ -4622,6 +5290,7 @@ impl KubernetesComputeDriver { let (tx, rx) = mpsc::channel(256); self.spawn_sandbox_runtime_periodic_reconcile(tx.clone()); let readiness_client = self.watch_client.clone(); + let readiness_shared = self.shared_supervisor.clone(); tokio::spawn(async move { let mut sandbox_name_to_id = std::collections::HashMap::::new(); @@ -4631,7 +5300,7 @@ impl KubernetesComputeDriver { tokio::select! { event = sandbox_stream.next() => match event { Some(Event::Apply(obj) | Event::InitApply(obj)) => { - if let Ok((kube_name, sandbox)) = sandbox_from_object_with_sandbox_runtime_readiness(&readiness_client, &namespace, obj).await { + if let Ok((kube_name, sandbox)) = sandbox_from_object_with_sandbox_runtime_readiness(&readiness_client, readiness_shared.as_ref(), &namespace, obj).await { update_indexes(&mut sandbox_name_to_id, &mut agent_pod_to_id, &kube_name, &sandbox); let event = WatchSandboxesEvent { payload: Some(watch_sandboxes_event::Payload::Sandbox( @@ -4725,6 +5394,30 @@ impl KubernetesComputeDriver { } } +/// Placement to release before a stop. An unreadable record must not block +/// suspending the workload, which ends any supervision on its own. +fn placement_for_release(object: &DynamicObject, sandbox_id: &str) -> SupervisorPlacement { + supervisor_placement(object).unwrap_or_else(|error| { + warn!(sandbox_id, %error, "invalid sandbox-runtime supervisor placement; stopping without releasing a sandbox instance"); + SupervisorPlacement::Dedicated + }) +} + +/// Shared generations that may currently run, with their sandbox instance +/// routes: the desired set of the shared supervisor. +fn runnable_shared_generations(sandboxes: &[DynamicObject]) -> Vec<(&DynamicObject, String)> { + sandboxes + .iter() + .filter(|object| { + sandbox_runtime_should_run(object) || sandbox_runtime_bootstrap_in_progress(object) + }) + .filter_map(|object| match supervisor_placement(object) { + Ok(SupervisorPlacement::Shared { route }) => Some((object, route)), + _ => None, + }) + .collect() +} + fn sandbox_runtime_bootstrap_completion_patch(resource_version: &str) -> serde_json::Value { serde_json::json!({ "metadata": { @@ -4741,33 +5434,120 @@ fn sandbox_runtime_bootstrap_completion_patch(resource_version: &str) -> serde_j }) } +/// Where a shared generation's launch material is persisted: the +/// generation's supervisor Secret, read back by recovery. +#[derive(Clone)] +struct SharedInstanceSecretRecord { + namespace: String, + sandbox_id: String, + generation: String, +} + +/// Launch material of one shared-supervisor sandbox instance, taken from the +/// generation's own Sandbox resource and gateway-issued credentials. The +/// coordinator validates, encodes, persists, and assigns it. +fn shared_instance_registration( + object: &DynamicObject, + route: &str, + sandbox_id: &str, + runtime_descriptor: openshell_sandbox_backend::boundary_protocol::SandboxRuntimeDescriptor, + launch_authentication: &openshell_core::jwt::SandboxLaunchAuthentication, + proxy_ca: crate::sandbox_runtime::ProxyCaMaterial, +) -> Result { + Ok(SandboxInstanceRegistration { + route: route.to_string(), + sandbox_id: sandbox_id.to_string(), + sandbox_name: annotation_or_label(object, LABEL_SANDBOX_NAME).ok_or_else(|| { + KubernetesDriverError::Precondition( + "Sandbox resource is missing its OpenShell sandbox name".to_string(), + ) + })?, + runtime_descriptor, + auth: launch_authentication.supervisor.clone(), + main_process_spec: required_sandbox_annotation( + object, + ANNOTATION_SANDBOX_RUNTIME_MAIN_PROCESS_SPEC, + )?, + workdir: Some(WORKSPACE_MOUNT_PATH.to_string()), + container_image: object + .data + .pointer("/spec/podTemplate/spec/containers") + .and_then(serde_json::Value::as_array) + .and_then(|containers| { + containers.iter().find(|container| { + container.get("name").and_then(serde_json::Value::as_str) == Some("agent") + }) + }) + .and_then(|container| container.get("image")) + .and_then(serde_json::Value::as_str) + .unwrap_or_default() + .to_string(), + proxy_ca: openshell_supervisor_control::ProxyCaMaterial { + certificate_pem: proxy_ca.certificate_pem, + private_key_pem: openshell_supervisor_control::SecretPem::new(proxy_ca.private_key_pem), + }, + trace_context: RegistrationTraceContext::current(), + }) +} + +/// What bootstrap completion waits for before the runtime is declared ready. +enum BootstrapControl { + /// The generation's dedicated supervisor Pod becomes Ready. + SupervisorPod { pods: Api, name: String }, + /// The gateway accepts the generation's shared-supervisor sandbox instance. + SharedInstance { + shared: SharedSupervisor, + route: String, + sandbox_id: String, + }, +} + +impl BootstrapControl { + async fn wait_available(self) -> bool { + match self { + Self::SupervisorPod { pods, name } => { + let available = |pod: Option<&Pod>| { + pod.is_some_and(|pod| { + sandbox_runtime_control_availability_from_pod(pod) + == SandboxRuntimeControlAvailability::Available + }) + }; + match await_condition(pods, &name, available).await { + Ok(Some(_)) => true, + Ok(None) => false, + Err(error) => { + debug!(%error, pod = name, "sandbox-runtime bootstrap availability watch failed; reconciliation will retry"); + false + } + } + } + // A disconnected or still-reconciling host is not a verdict; the + // coordinator keeps waiting within the bootstrap grace. + Self::SharedInstance { + shared, + route, + sandbox_id, + } => { + shared + .supervision() + .wait_for_ready(&route, &sandbox_id) + .await + } + } + } +} + fn spawn_sandbox_runtime_bootstrap_completion( - pods: Api, + control: BootstrapControl, sandboxes: Api, - supervisor_pod_name: String, sandbox_name: String, expected_sandbox_uid: Option, ) { tokio::spawn(async move { let deadline = tokio::time::Instant::now() + SANDBOX_RUNTIME_BOOTSTRAP_GRACE; - let available = |pod: Option<&Pod>| { - pod.is_some_and(|pod| { - sandbox_runtime_control_availability_from_pod(pod) - == SandboxRuntimeControlAvailability::Available - }) - }; - match tokio::time::timeout_at( - deadline, - await_condition(pods, &supervisor_pod_name, available), - ) - .await - { - Ok(Ok(Some(_))) => {} - Ok(Ok(None)) | Err(_) => return, - Ok(Err(error)) => { - debug!(%error, sandbox = sandbox_name, "sandbox-runtime bootstrap availability watch failed; reconciliation will retry"); - return; - } + match tokio::time::timeout_at(deadline, control.wait_available()).await { + Ok(true) => {} + Ok(false) | Err(_) => return, } // A replacement control can become Available before the Agent @@ -4944,6 +5724,7 @@ where { let (tx, rx) = mpsc::channel(256); driver.spawn_sandbox_runtime_periodic_reconcile(tx.clone()); + let readiness_shared = driver.shared_supervisor; tokio::spawn(async move { loop { @@ -4952,7 +5733,7 @@ where Some(Event::Apply(obj) | Event::InitApply(obj)) => { let ns = obj.metadata.namespace.clone() .unwrap_or_else(|| default_namespace.clone()); - if let Ok((_kube_name, sandbox)) = sandbox_from_object_with_sandbox_runtime_readiness(&readiness_client, &ns, obj).await { + if let Ok((_kube_name, sandbox)) = sandbox_from_object_with_sandbox_runtime_readiness(&readiness_client, readiness_shared.as_ref(), &ns, obj).await { let event = WatchSandboxesEvent { payload: Some(watch_sandboxes_event::Payload::Sandbox( WatchSandboxesSandboxEvent { sandbox: Some(sandbox) } @@ -5697,6 +6478,94 @@ async fn sandbox_runtime_control_availability( combine_sandbox_runtime_control_availability([control, service, fence, supervisor_fence]) } +/// Availability of a generation's supervision, by its recorded placement. +async fn generation_control_availability( + client: &Client, + shared: Option<&SharedSupervisor>, + namespace: &str, + sandbox_id: &str, + object: &DynamicObject, +) -> SandboxRuntimeControlAvailability { + match supervisor_placement(object) { + Ok(SupervisorPlacement::Dedicated) => { + sandbox_runtime_control_availability(client, namespace, sandbox_id).await + } + Ok(SupervisorPlacement::Shared { route }) => { + shared_instance_control_availability(client, shared, namespace, sandbox_id, &route) + .await + } + Err(error) => { + warn!(sandbox_id, %error, "sandbox-runtime supervisor placement is invalid"); + SandboxRuntimeControlAvailability::Unavailable + } + } +} + +/// A shared generation is supervised while its sandbox instance is registered +/// for this sandbox and accepted by the gateway, and its fence and egress +/// allowance exist. +async fn shared_instance_control_availability( + client: &Client, + shared: Option<&SharedSupervisor>, + namespace: &str, + sandbox_id: &str, + route: &str, +) -> SandboxRuntimeControlAvailability { + let Some(shared) = shared else { + warn!( + sandbox_id, + "the gateway supervisor topology is not configured for a shared generation" + ); + return SandboxRuntimeControlAvailability::Unavailable; + }; + let names = SandboxRuntimeNames::new(sandbox_id); + let policies = Api::::namespaced(client.clone(), namespace); + let exists = |result: Result, KubeError>, _>, what: &str| { + match result { + Ok(Ok(Some(_))) => SandboxRuntimeControlAvailability::Available, + Ok(Ok(None)) => SandboxRuntimeControlAvailability::Unavailable, + Ok(Err(error)) => { + warn!(sandbox_id, %error, "could not determine sandbox-runtime {what} availability"); + SandboxRuntimeControlAvailability::Unknown + } + Err(tokio::time::error::Elapsed { .. }) => { + warn!( + sandbox_id, + "timed out checking sandbox-runtime {what} availability" + ); + SandboxRuntimeControlAvailability::Unknown + } + } + }; + // The coordinator renders the host's verdict; a disconnected or + // still-reconciling host is not one and must not suspend a running sandbox. + let instance = match shared.supervision().supervision(route, sandbox_id) { + InstanceSupervision::Available => SandboxRuntimeControlAvailability::Available, + InstanceSupervision::Degraded => SandboxRuntimeControlAvailability::Degraded, + InstanceSupervision::Unavailable => SandboxRuntimeControlAvailability::Unavailable, + InstanceSupervision::Unknown => { + warn!( + sandbox_id, + "no shared supervisor host is connected; sandbox instance availability is unknown" + ); + SandboxRuntimeControlAvailability::Unknown + } + }; + let (fence, control) = tokio::join!( + tokio::time::timeout(KUBE_API_TIMEOUT, policies.get_opt(&names.workload_policy)), + tokio::time::timeout( + KUBE_API_TIMEOUT, + policies.get_opt(&names.shared_control_policy) + ), + ); + combine_sandbox_runtime_control_availability([ + instance, + exists(fence, "workload fence"), + exists(control, "shared control policy"), + SandboxRuntimeControlAvailability::Available, + ]) +} + fn combine_sandbox_runtime_control_availability( dependencies: [SandboxRuntimeControlAvailability; 4], ) -> SandboxRuntimeControlAvailability { @@ -5762,6 +6631,14 @@ async fn sandbox_runtime_supervisor_generation_matches( sandbox_id: &str, sandbox: &DynamicObject, ) -> SandboxRuntimeControlAvailability { + match supervisor_placement(sandbox) { + Ok(SupervisorPlacement::Dedicated) => {} + // The sandbox instance's sandbox is checked with its availability. + Ok(SupervisorPlacement::Shared { .. }) => { + return SandboxRuntimeControlAvailability::Available; + } + Err(_) => return SandboxRuntimeControlAvailability::Unavailable, + } let Some(expected_uid) = sandbox .metadata .annotations @@ -5793,6 +6670,7 @@ async fn sandbox_runtime_supervisor_generation_matches( async fn sandbox_from_object_with_sandbox_runtime_readiness( client: &Client, + shared: Option<&SharedSupervisor>, namespace: &str, obj: DynamicObject, ) -> Result<(String, Sandbox), String> { @@ -5808,10 +6686,12 @@ async fn sandbox_from_object_with_sandbox_runtime_readiness( mark_sandbox_runtime_bootstrapping(&mut sandbox); } if !sandbox_id.is_empty() { - let dependencies = Box::pin(sandbox_runtime_control_availability( + let dependencies = Box::pin(generation_control_availability( client, + shared, &object_namespace, &sandbox_id, + &obj, )); let workload_generation = Box::pin(sandbox_runtime_workload_generation_matches( client, @@ -7508,6 +8388,8 @@ fn sandbox_runtime_suspension_completion_patch(resource_version: &str) -> serde_ ANNOTATION_SANDBOX_RUNTIME_GENERATION: serde_json::Value::Null, ANNOTATION_SANDBOX_RUNTIME_WORKLOAD_UID: serde_json::Value::Null, ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID: serde_json::Value::Null, + ANNOTATION_SUPERVISOR_TOPOLOGY: serde_json::Value::Null, + ANNOTATION_SUPERVISOR_SESSION: serde_json::Value::Null, ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_UID: serde_json::Value::Null, ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_GENERATION: serde_json::Value::Null, ANNOTATION_SANDBOX_RUNTIME_READINESS: "unavailable", @@ -8083,6 +8965,7 @@ mod tests { lifecycle_gates: Arc::default(), config: KubernetesComputeConfig::default(), operator_allowlist: None, + shared_supervisor: None, }; driver .sandbox_api_version @@ -8663,6 +9546,66 @@ mod tests { sandbox } + #[test] + fn only_runnable_shared_generations_keep_their_instances() { + let shared = |route: &str, operating_mode: &str, annotations: &[(&str, &str)]| { + let mut object = sandbox_object_for_test(route, route); + object.data = serde_json::json!({"spec": {"operatingMode": operating_mode}}); + let mut recorded = BTreeMap::from([ + ( + ANNOTATION_SUPERVISOR_TOPOLOGY.to_string(), + GATEWAY_TOPOLOGY.to_string(), + ), + (ANNOTATION_SUPERVISOR_SESSION.to_string(), route.to_string()), + ]); + recorded.extend( + annotations + .iter() + .map(|(key, value)| ((*key).to_string(), (*value).to_string())), + ); + object.metadata.annotations = Some(recorded); + object + }; + let running = "rt-running-route-0001"; + let stopping = "rt-stopping-route-0001"; + let stopped = "rt-stopped-route-00001"; + let mut dedicated = sandbox_object_for_test("dedicated", "dedicated"); + dedicated.data = serde_json::json!({"spec": {"operatingMode": "Running"}}); + let sandboxes = [ + shared(running, "Running", &[]), + // A stop in progress still ends its sandbox instance itself. + shared( + stopping, + "Suspended", + &[(ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAPPING, "true")], + ), + shared(stopped, "Suspended", &[]), + dedicated, + ]; + let routes = runnable_shared_generations(&sandboxes) + .into_iter() + .map(|(_, route)| route) + .collect::>(); + assert_eq!( + routes, + HashSet::from([running.to_string(), stopping.to_string()]) + ); + } + + #[test] + fn suspension_completion_forgets_the_generation_placement() { + let patch = sandbox_runtime_suspension_completion_patch("7"); + let annotations = &patch["metadata"]["annotations"]; + assert!(annotations[ANNOTATION_SUPERVISOR_TOPOLOGY].is_null()); + assert!(annotations[ANNOTATION_SUPERVISOR_SESSION].is_null()); + assert!( + annotations + .as_object() + .unwrap() + .contains_key(ANNOTATION_SUPERVISOR_SESSION) + ); + } + #[test] fn restart_runtime_selection_preserves_namespace_and_resource_uid() { let expected = parse_kubernetes_runtime_identity( @@ -9104,6 +10047,7 @@ mod tests { lifecycle_gates: Arc::default(), config: KubernetesComputeConfig::default(), operator_allowlist: None, + shared_supervisor: None, }; driver .sandbox_api_version @@ -9195,6 +10139,7 @@ mod tests { lifecycle_gates: Arc::default(), config: KubernetesComputeConfig::default(), operator_allowlist: None, + shared_supervisor: None, }; driver @@ -11934,6 +12879,7 @@ mod tests { lifecycle_gates: Arc::default(), config, operator_allowlist: None, + shared_supervisor: None, }; (driver, steps, bodies) } @@ -12238,6 +13184,7 @@ mod tests { lifecycle_gates: Arc::default(), config: managed_config(&[]), operator_allowlist: None, + shared_supervisor: None, }; let service = crate::ComputeDriverService::new(driver); diff --git a/crates/openshell-driver-kubernetes/src/isolation.rs b/crates/openshell-driver-kubernetes/src/isolation.rs index c9da7f3e3a..acab3ca03c 100644 --- a/crates/openshell-driver-kubernetes/src/isolation.rs +++ b/crates/openshell-driver-kubernetes/src/isolation.rs @@ -53,6 +53,10 @@ struct KubernetesControlPolicyEvidence<'a> { network_policy_uid: &'a str, network_policy_resource_version: &'a str, port: u16, + /// Namespace of a supervisor shared by sandboxes. Omitted for a + /// dedicated supervisor so its evidence encoding is unchanged. + #[serde(skip_serializing_if = "Option::is_none")] + supervisor_namespace: Option<&'a str>, } impl KubernetesControlPolicyEvidence<'_> { @@ -281,6 +285,68 @@ pub fn runtime_control_policy(namespace: &str, name: &str, pair: &str, port: u16 } } +/// Per-sandbox allowance for runtime-initiated control to a shared supervisor. +/// +/// It selects only this pair's workload Pod and admits one egress flow, to the +/// shared supervisor Pods in their namespace on the runtime port. The +/// namespace fence keeps denying every other workload connection. Which +/// sandbox instance a connection reaches is decided by its routing preface +/// and pinned TLS identity, not by this policy. +#[must_use] +pub fn shared_runtime_control_policy( + namespace: &str, + name: &str, + pair: &str, + supervisor_namespace: &str, + supervisor_selector: BTreeMap, + port: u16, +) -> NetworkPolicy { + NetworkPolicy { + metadata: ObjectMeta { + name: Some(name.to_string()), + namespace: Some(namespace.to_string()), + ..Default::default() + }, + spec: Some(NetworkPolicySpec { + pod_selector: LabelSelector { + match_labels: Some(pair_role_labels(pair, WORKLOAD_ROLE)), + ..Default::default() + }, + policy_types: Some(vec!["Egress".to_string()]), + egress: Some(vec![NetworkPolicyEgressRule { + to: Some(vec![NetworkPolicyPeer { + namespace_selector: Some(LabelSelector { + match_labels: Some(BTreeMap::from([( + "kubernetes.io/metadata.name".to_string(), + supervisor_namespace.to_string(), + )])), + ..Default::default() + }), + pod_selector: Some(LabelSelector { + match_labels: Some(supervisor_selector), + ..Default::default() + }), + ..Default::default() + }]), + ports: Some(vec![NetworkPolicyPort { + port: Some(IntOrString::Int(i32::from(port))), + protocol: Some("TCP".to_string()), + ..Default::default() + }]), + }]), + ..Default::default() + }), + } +} + +/// The supervisor bound into a generation's resource claims. +pub enum KubernetesSupervisorClaim { + /// UID of the generation's dedicated supervisor Pod. + Pod(String), + /// Route of the generation's sandbox instance on the shared supervisor. + SandboxInstance(String), +} + /// Which side opens the Sandbox Protocol connection, with the verified /// Kubernetes coordinates that mode needs. pub enum KubernetesControlPath { @@ -301,6 +367,18 @@ pub enum KubernetesControlPath { control_policy_uid: String, control_policy_resource_version: String, }, + /// The workload runtime dials the gateway's shared supervisor and names + /// this generation's route before TLS. + RuntimeInitiatedShared { + control_authority: String, + /// Shared Service address the runtime dials. + supervisor_address: SocketAddr, + supervisor_namespace: String, + route: String, + /// Verified per-sandbox shared control policy coordinates. + control_policy_uid: String, + control_policy_resource_version: String, + }, } /// Driver-owned inputs that bind one workload/supervisor pair to one boundary. @@ -321,7 +399,7 @@ pub struct KubernetesSandboxRuntimeBoundarySpec { pub sandbox_resource_uid: String, pub workload_pod_uid: String, pub workload_pod_uid_path: PathBuf, - pub supervisor_pod_uid: String, + pub supervisor: KubernetesSupervisorClaim, pub egress_policy_uid: String, pub egress_policy_resource_version: String, pub control_path: KubernetesControlPath, @@ -352,10 +430,15 @@ impl KubernetesSandboxRuntimeBoundarySpec { "kubernetes.workload_pod_uid".to_string(), self.workload_pod_uid, ), - ( - "kubernetes.supervisor_pod_uid".to_string(), - self.supervisor_pod_uid, - ), + match self.supervisor { + KubernetesSupervisorClaim::Pod(uid) => { + ("kubernetes.supervisor_pod_uid".to_string(), uid) + } + // The claim key keeps its established `supervisor_session` name. + KubernetesSupervisorClaim::SandboxInstance(route) => { + ("kubernetes.supervisor_session".to_string(), route) + } + }, ( "kubernetes.egress_policy_uid".to_string(), self.egress_policy_uid, @@ -365,6 +448,8 @@ impl KubernetesSandboxRuntimeBoundarySpec { self.egress_policy_resource_version, ), ]); + let mut supervisor_namespace = None; + let mut shared_port = None; let (listener, transport) = match self.control_path { KubernetesControlPath::SupervisorInitiated { boundary_listener, @@ -400,6 +485,7 @@ impl KubernetesSandboxRuntimeBoundarySpec { authority: control_authority.clone(), addresses: vec![supervisor_address], tls: self.sandbox_tls, + route: None, }, SandboxTransport::RuntimeInitiatedTcp { authority: control_authority, @@ -407,11 +493,43 @@ impl KubernetesSandboxRuntimeBoundarySpec { }, ) } + KubernetesControlPath::RuntimeInitiatedShared { + control_authority, + supervisor_address, + supervisor_namespace: shared_namespace, + route, + control_policy_uid, + control_policy_resource_version, + } => { + resource_claims.insert( + "kubernetes.control_policy_uid".to_string(), + control_policy_uid, + ); + resource_claims.insert( + "kubernetes.control_policy_resource_version".to_string(), + control_policy_resource_version, + ); + supervisor_namespace = Some(shared_namespace); + shared_port = Some(supervisor_address.port()); + ( + BoundaryListener::RuntimeInitiatedTlsTcp { + authority: control_authority.clone(), + addresses: vec![supervisor_address], + tls: self.sandbox_tls, + route: Some(route.clone()), + }, + SandboxTransport::RuntimeInitiatedRouted { + authority: control_authority, + route, + }, + ) + } }; let control_port = match &transport { SandboxTransport::RuntimeInitiatedTcp { listen_address, .. } => { Some(listen_address.port()) } + SandboxTransport::RuntimeInitiatedRouted { .. } => shared_port, _ => None, }; let provisioning_evidence = KubernetesOuterFenceEvidence { @@ -426,6 +544,7 @@ impl KubernetesSandboxRuntimeBoundarySpec { network_policy_resource_version: &resource_claims ["kubernetes.control_policy_resource_version"], port, + supervisor_namespace: supervisor_namespace.as_deref(), }), } .project(&self.generation)?; @@ -531,7 +650,7 @@ mod tests { sandbox_resource_uid: "sandbox-resource-uid".to_string(), workload_pod_uid: "pod-uid".to_string(), workload_pod_uid_path: PathBuf::from("/.openshell/pod-identity/uid"), - supervisor_pod_uid: "supervisor-pod-uid".to_string(), + supervisor: KubernetesSupervisorClaim::Pod("supervisor-pod-uid".to_string()), egress_policy_uid: "network-policy-uid".to_string(), egress_policy_resource_version: "1945".to_string(), control_path: KubernetesControlPath::SupervisorInitiated { diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index 963935abef..4d21fbff51 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -9,12 +9,13 @@ mod lifecycle; pub mod otel_tracing; mod resource_admission; mod sandbox_runtime; +mod shared_supervisor; pub use config::{ DEFAULT_GATEWAY_ID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, KubernetesControlConnection, KubernetesImagePullPolicy, - KubernetesSandboxRuntimeConfig, ManagedSshIngressConfig, SupervisorTopology, WorkspaceMode, - managed_namespace_prefix, + KubernetesSandboxRuntimeConfig, ManagedSshIngressConfig, SharedSupervisorConfig, + SupervisorTopology, WorkspaceMode, managed_namespace_prefix, }; pub use driver::{KubernetesComputeDriver, KubernetesDriverError}; pub use grpc::ComputeDriverService; diff --git a/crates/openshell-driver-kubernetes/src/lifecycle_tests.rs b/crates/openshell-driver-kubernetes/src/lifecycle_tests.rs index 16c8f82981..d8ec7791c0 100644 --- a/crates/openshell-driver-kubernetes/src/lifecycle_tests.rs +++ b/crates/openshell-driver-kubernetes/src/lifecycle_tests.rs @@ -36,6 +36,7 @@ mod lifecycle_reconciliation { lifecycle_gates: Arc::default(), config: KubernetesComputeConfig::default(), operator_allowlist: None, + shared_supervisor: None, }; (driver, steps, Arc::default()) } diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 30c7f899e7..209d286370 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -261,6 +261,7 @@ async fn main() -> Result<()> { let driver = KubernetesComputeDriver::new( KubernetesComputeConfig { supervisor_topology: None, + shared_supervisor: None, allow_driver_config: args.admission_config_json.allow_driver_config, resource_admission: args.admission_config_json.resource_admission.clone(), workspace_mode: args.workspace_mode, diff --git a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs index d191ca5201..09843f1d74 100644 --- a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs +++ b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs @@ -43,6 +43,8 @@ pub const PROXY_CA_PRIVATE_KEY: &str = "proxy-ca.key"; /// bundle (`proxy_ca_bundle`). Distinct from [`PROXY_CA_CERTIFICATE_KEY`], /// which is the sandbox's own generated TLS-interception CA. pub const UPSTREAM_PROXY_CA_BUNDLE_KEY: &str = "upstream-proxy-ca.pem"; +/// JSON sandbox instance registration of a generation on the shared supervisor. +pub const SHARED_INSTANCE_KEY: &str = "instance.json"; pub const CLIENT_TLS_CA_KEY: &str = "client-ca.crt"; pub const SANDBOX_BOOTSTRAP_INPUT_PATH: &str = "/.openshell/bootstrap-input"; pub const BOUNDARY_CONFIG_PATH: &str = "/.openshell/state/bootstrap/boundary.json"; @@ -118,6 +120,8 @@ pub struct SandboxRuntimeNames { pub supervisor_service: String, /// Per-sandbox egress allowance used only by runtime-initiated control. pub control_policy: String, + /// Per-sandbox egress allowance to the gateway's shared supervisor. + pub shared_control_policy: String, pub supervisor_pod: String, pub workload_policy: String, pub supervisor_policy: String, @@ -134,6 +138,7 @@ impl SandboxRuntimeNames { boundary_service: format!("os-boundary-{suffix}"), supervisor_service: format!("os-supervisor-{suffix}"), control_policy: format!("os-control-{suffix}"), + shared_control_policy: format!("os-shared-control-{suffix}"), supervisor_pod: format!("os-supervisor-{suffix}"), workload_policy: NAMESPACE_WORKLOAD_POLICY_NAME.to_string(), supervisor_policy: NAMESPACE_SUPERVISOR_EGRESS_POLICY_NAME.to_string(), @@ -283,6 +288,30 @@ pub fn control_policy( policy } +/// Egress allowance from this sandbox's workload Pod to the shared supervisor. +#[must_use] +pub fn shared_control_policy( + namespace: &str, + names: &SandboxRuntimeNames, + sandbox_id: &str, + supervisor_namespace: &str, + supervisor_selector: BTreeMap, + runtime_port: u16, + owner: OwnerReference, +) -> NetworkPolicy { + let mut policy = crate::isolation::shared_runtime_control_policy( + namespace, + &names.shared_control_policy, + &pair_label_value(sandbox_id), + supervisor_namespace, + supervisor_selector, + runtime_port, + ); + policy.metadata.owner_references = Some(vec![owner]); + policy.metadata.labels = Some(common_labels(sandbox_id, "sandbox-shared-control-egress")); + policy +} + #[allow(clippy::too_many_arguments, clippy::similar_names)] pub fn supervisor_pod( namespace: &str, @@ -661,6 +690,35 @@ pub fn supervisor_bootstrap_secret( } } +/// Durable launch material of one shared-supervisor sandbox instance, under +/// the generation's supervisor Secret name so the existing generation cleanup +/// deletes it. The gateway reads it back to rebuild the host's assignments. +#[must_use] +pub fn shared_instance_secret( + namespace: &str, + names: &SandboxRuntimeNames, + sandbox_id: &str, + registration: Vec, + owner: OwnerReference, +) -> Secret { + Secret { + metadata: ObjectMeta { + name: Some(names.supervisor_secret.clone()), + namespace: Some(namespace.to_string()), + owner_references: Some(vec![owner]), + labels: Some(common_labels(sandbox_id, SUPERVISOR_SECRET_COMPONENT)), + ..Default::default() + }, + data: Some(BTreeMap::from([( + SHARED_INSTANCE_KEY.to_string(), + ByteString(registration), + )])), + immutable: Some(true), + type_: Some("Opaque".to_string()), + ..Default::default() + } +} + /// Copy a configured image-pull Secret into an immutable Secret for one /// runtime generation. #[must_use] diff --git a/crates/openshell-driver-kubernetes/src/shared_supervisor.rs b/crates/openshell-driver-kubernetes/src/shared_supervisor.rs new file mode 100644 index 0000000000..850e261e34 --- /dev/null +++ b/crates/openshell-driver-kubernetes/src/shared_supervisor.rs @@ -0,0 +1,221 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Generations supervised by the gateway's shared supervisor. +//! +//! Each runtime generation records where its supervisor runs when it starts. +//! Every later lifecycle decision reads that record, so changing the gateway +//! topology setting never reinterprets an existing generation. +//! +//! The gateway owns which sandbox instances the shared supervisor hosts and +//! coordinates them through `openshell_supervisor_control::SharedSupervision`, +//! which it constructs and hands to this driver. The driver's part is +//! Kubernetes: the placement record on the Sandbox resource, the launch +//! Secret the coordinator persists launch material into and reads it back +//! from, the egress allowance from the workload to the supervisor's Service, +//! and delivery of the gateway-minted supervisor token into the Secret the +//! supervisor Pod mounts. + +use std::collections::BTreeMap; +use std::time::Duration; + +use k8s_openapi::api::core::v1::Secret; +use kube::api::{Api, ObjectMeta, Patch, PatchParams}; +use kube::core::DynamicObject; +use openshell_supervisor_control::{SharedSupervision, SupervisorCredential, deliver_credentials}; +use tracing::info; + +use crate::config::SharedSupervisorConfig; +use crate::driver::KubernetesDriverError; + +/// Supervisor placement of the current generation; absent means a dedicated +/// supervisor Pod. +pub const ANNOTATION_SUPERVISOR_TOPOLOGY: &str = "openshell.ai/sandbox-runtime-supervisor-topology"; +/// Route of a generation's sandbox instance on the shared supervisor. The +/// annotation key keeps its established `-session` name. +pub const ANNOTATION_SUPERVISOR_SESSION: &str = "openshell.ai/sandbox-runtime-supervisor-session"; +pub const GATEWAY_TOPOLOGY: &str = "gateway"; +/// Key of the supervisor token in `shared_supervisor.token_secret_name`. +pub const TOKEN_SECRET_KEY: &str = "token"; +/// Annotation recording the token's expiry (Unix seconds) for operators. +pub const ANNOTATION_TOKEN_EXPIRES_AT: &str = "openshell.ai/supervisor-token-expires-at"; +const TOKEN_WRITE_RETRY: Duration = Duration::from_secs(5); + +/// Where a generation's supervisor runs, as recorded when it started. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SupervisorPlacement { + /// A supervisor Pod paired with this generation's workload Pod. + Dedicated, + /// A sandbox instance on the gateway's shared supervisor. + Shared { route: String }, +} + +/// Read the recorded placement. An unknown or incomplete record fails closed. +pub fn supervisor_placement( + object: &DynamicObject, +) -> Result { + let annotations = object.metadata.annotations.as_ref(); + let annotation = |key: &str| annotations.and_then(|annotations| annotations.get(key)); + match annotation(ANNOTATION_SUPERVISOR_TOPOLOGY).map(String::as_str) { + None => Ok(SupervisorPlacement::Dedicated), + Some(GATEWAY_TOPOLOGY) => { + let route = annotation(ANNOTATION_SUPERVISOR_SESSION) + .filter(|route| { + openshell_sandbox_backend::runtime_routed::validate_route_handle(route).is_ok() + }) + .ok_or_else(|| { + KubernetesDriverError::Precondition( + "shared supervisor generation has no valid sandbox instance route" + .to_string(), + ) + })?; + Ok(SupervisorPlacement::Shared { + route: route.clone(), + }) + } + Some(other) => Err(KubernetesDriverError::Precondition(format!( + "unknown sandbox-runtime supervisor topology {other:?}" + ))), + } +} + +/// The shared supervisor as this driver reaches it: the Helm-deployed +/// Service and token Secret it is configured with, and the gateway's +/// coordination of its sandbox instances. +#[derive(Clone, Debug)] +pub struct SharedSupervisor { + config: SharedSupervisorConfig, + supervision: SharedSupervision, +} + +impl SharedSupervisor { + pub fn new(config: SharedSupervisorConfig, supervision: SharedSupervision) -> Self { + Self { + config, + supervision, + } + } + + /// Keep the supervisor token Secret current with every token the gateway + /// publishes, until shutdown. The Secret itself is deployment packaging + /// (the chart creates it empty); the driver only replaces its token. + pub fn spawn_token_secret_writer( + &self, + client: kube::Client, + shutdown_rx: tokio::sync::watch::Receiver, + ) { + let secrets = Api::::namespaced(client, &self.config.service_namespace); + let name = self.config.token_secret_name.clone(); + tokio::spawn(deliver_credentials( + self.supervision.credentials(), + shutdown_rx, + TOKEN_WRITE_RETRY, + move |credential| { + let secrets = secrets.clone(); + let name = name.clone(); + async move { write_token_secret(&secrets, &name, &credential).await } + }, + )); + } + + pub fn config(&self) -> &SharedSupervisorConfig { + &self.config + } + + /// The gateway's coordination of the host's sandbox instances. + pub fn supervision(&self) -> &SharedSupervision { + &self.supervision + } +} + +/// Replace the token in the chart-created Secret. The error names what an +/// operator must restore when the write cannot succeed on its own. +async fn write_token_secret( + secrets: &Api, + name: &str, + credential: &SupervisorCredential, +) -> Result<(), String> { + let secret = Secret { + metadata: ObjectMeta { + annotations: Some(BTreeMap::from([( + ANNOTATION_TOKEN_EXPIRES_AT.to_string(), + credential.expires_at.to_string(), + )])), + ..ObjectMeta::default() + }, + string_data: Some(BTreeMap::from([( + TOKEN_SECRET_KEY.to_string(), + credential.token.expose_secret().to_string(), + )])), + ..Secret::default() + }; + secrets + .patch(name, &PatchParams::default(), &Patch::Merge(&secret)) + .await + .map_err(|error| match &error { + kube::Error::Api(response) if response.code == 404 => format!( + "the supervisor token Secret {name} does not exist; the chart creates it, \ + restore it (helm upgrade) so the shared supervisor can authenticate" + ), + kube::Error::Api(response) if response.code == 403 => format!( + "the gateway may not patch the supervisor token Secret {name}; check the \ + chart's gateway Role" + ), + _ => format!("failed to write the supervisor token Secret {name}: {error}"), + })?; + info!( + secret = %name, + expires_at = credential.expires_at, + "wrote the supervisor token Secret" + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use kube::core::{ApiResource, GroupVersionKind}; + + fn object(annotations: &[(&str, &str)]) -> DynamicObject { + let resource = ApiResource::from_gvk(&GroupVersionKind::gvk( + "agents.x-k8s.io", + "v1beta1", + "Sandbox", + )); + let mut object = DynamicObject::new("sandbox-a", &resource); + object.metadata.annotations = Some( + annotations + .iter() + .map(|(key, value)| ((*key).to_string(), (*value).to_string())) + .collect(), + ); + object + } + + #[test] + fn recorded_placement_is_read_and_unknown_records_fail_closed() { + assert_eq!( + supervisor_placement(&object(&[])).unwrap(), + SupervisorPlacement::Dedicated + ); + let route = openshell_sandbox_backend::runtime_routed::new_route_handle(); + assert_eq!( + supervisor_placement(&object(&[ + (ANNOTATION_SUPERVISOR_TOPOLOGY, GATEWAY_TOPOLOGY), + (ANNOTATION_SUPERVISOR_SESSION, &route), + ])) + .unwrap(), + SupervisorPlacement::Shared { route } + ); + for annotations in [ + &[(ANNOTATION_SUPERVISOR_TOPOLOGY, GATEWAY_TOPOLOGY)][..], + &[ + (ANNOTATION_SUPERVISOR_TOPOLOGY, GATEWAY_TOPOLOGY), + (ANNOTATION_SUPERVISOR_SESSION, "Not/A/Route"), + ], + &[(ANNOTATION_SUPERVISOR_TOPOLOGY, "workspace")], + ] { + assert!(supervisor_placement(&object(annotations)).is_err()); + } + } +} diff --git a/crates/openshell-gateway/src/lib.rs b/crates/openshell-gateway/src/lib.rs index d353cb4f1b..3f060ae9f7 100644 --- a/crates/openshell-gateway/src/lib.rs +++ b/crates/openshell-gateway/src/lib.rs @@ -285,11 +285,23 @@ impl openshell_server::ComputeDriverFactory for KubernetesFactory { .map(str::to_string); config.supervisor_ocsf_export = supervisor_ocsf_export(context.otlp_config()); config.supervisor_gateway_name = Some(context.gateway_name().to_string()); - let driver = openshell_driver_kubernetes::KubernetesComputeDriver::new( - config, - context.shutdown_receiver(), - ) - .await + // With the gateway supervisor topology, the gateway owns the shared + // supervisor's assignments and token; the driver adapts them to + // Kubernetes objects. + let driver = if config.shared_supervisor.is_some() { + openshell_driver_kubernetes::KubernetesComputeDriver::with_shared_supervision( + config, + context.shutdown_receiver(), + context.shared_supervision(), + ) + .await + } else { + openshell_driver_kubernetes::KubernetesComputeDriver::new( + config, + context.shutdown_receiver(), + ) + .await + } .map_err(|error| openshell_core::Error::execution(error.to_string()))?; let driver = openshell_driver_kubernetes::ComputeDriverService::new_in_process(driver); Ok(openshell_server::ComputeDriverInstance::InProcess( diff --git a/crates/openshell-sandbox-backend/README.md b/crates/openshell-sandbox-backend/README.md index a63d52ebde..9045a8edf8 100644 --- a/crates/openshell-sandbox-backend/README.md +++ b/crates/openshell-sandbox-backend/README.md @@ -124,6 +124,23 @@ open, so reconnects and credential rotation never wait for a new dial. The new variants leave existing encodings unchanged. Binaries that predate them reject runtime-initiated descriptors instead of falling back to another mode. +`SandboxTransport::RuntimeInitiatedRouted` serves the sandbox instances of a +supervisor shared by many sandboxes from one listener +(`runtime_routed::RuntimeRouter`). A sandbox instance is one running +generation of a stable Sandbox resource. The runtime first sends a fixed +preface: `OSRP`, version `1`, a length byte, and an opaque route handle of 16 +to 64 lowercase letters, digits, or hyphens. The handle is untrusted; it only +selects which instance's generation-pinned TLS identity the supervisor then +demands. Unknown, removed, and replaced handles are closed before any +handshake. Handshakes are bounded globally and per route, and an +authenticated connection only fills its own instance's spare queue. A routed +instance waits up to five minutes for its first connection, since it is +registered before its workload Pod is scheduled, and the usual connect window +afterwards. The launch material and status of one hosted instance +(`SandboxInstanceRegistration`, `SandboxInstanceStatusReport`) are defined by +`openshell-supervisor-control`, which depends on this crate for the runtime +descriptor and route validation. + Runtime-initiated transport supports TCP, protected Unix sockets, and host vsock listeners. These share bounded TLS handshakes, the authenticated spare queue, and the same generation-pinned runtime identity check. The legacy diff --git a/crates/openshell-sandbox-backend/src/boundary_protocol.rs b/crates/openshell-sandbox-backend/src/boundary_protocol.rs index cf772b7118..8402a5e7a5 100644 --- a/crates/openshell-sandbox-backend/src/boundary_protocol.rs +++ b/crates/openshell-sandbox-backend/src/boundary_protocol.rs @@ -269,6 +269,14 @@ pub enum SandboxTransport { guest_cid: u32, port: u32, }, + /// The runtime dials a supervisor shared by several sandbox instances and + /// names this instance's opaque route handle before the pinned TLS + /// handshake. + /// Only a host that owns the shared listener can serve this transport. + RuntimeInitiatedRouted { + authority: String, + route: String, + }, } impl SandboxTransport { @@ -294,6 +302,9 @@ impl SandboxTransport { Self::Vsock { guest_cid, port } | Self::RuntimeInitiatedVsock { guest_cid, port } => { *guest_cid >= 3 && *guest_cid != u32::MAX && *port != 0 && *port != u32::MAX } + Self::RuntimeInitiatedRouted { authority, route } => { + !authority.is_empty() && crate::runtime_routed::validate_route_handle(route).is_ok() + } }; if valid { Ok(()) @@ -444,6 +455,10 @@ pub enum BoundaryListener { /// Driver-resolved supervisor endpoints, tried in order. addresses: Vec, tls: SandboxTlsServerConfig, + /// Route handle sent before TLS to a supervisor shared by several + /// sandbox instances. Absent for a dedicated supervisor. + #[serde(default, skip_serializing_if = "Option::is_none")] + route: Option, }, /// Dial the supervisor over the driver's protected shared Unix socket. RuntimeInitiatedUnix { diff --git a/crates/openshell-sandbox-backend/src/lib.rs b/crates/openshell-sandbox-backend/src/lib.rs index e38e216d02..6e1bc80872 100644 --- a/crates/openshell-sandbox-backend/src/lib.rs +++ b/crates/openshell-sandbox-backend/src/lib.rs @@ -18,6 +18,7 @@ pub mod mediation; pub mod protobuf; mod runtime; mod runtime_initiated; +pub mod runtime_routed; pub mod sandbox_auth; pub mod stream_adapter; @@ -27,7 +28,7 @@ pub use runtime::{ ConfirmedBoundary, ExecSession, NetworkMediationSource, OpenShellRuntimeBackend, ProcessAttachment, ReadyBoundary, RunningBoundary, }; -pub use runtime::{BoundaryTransport, BoundaryTransportConnector}; +pub use runtime::{BoundaryTransport, BoundaryTransportConnector, validate_runtime_coordinates}; /// Stable isolation backend name implemented by `openshell-sandbox`. pub const BACKEND_NAME: &str = openshell_core::isolation_registration::BUILTIN_BACKEND_NAME; diff --git a/crates/openshell-sandbox-backend/src/runtime.rs b/crates/openshell-sandbox-backend/src/runtime.rs index bc687e1b80..8e54aa9832 100644 --- a/crates/openshell-sandbox-backend/src/runtime.rs +++ b/crates/openshell-sandbox-backend/src/runtime.rs @@ -50,6 +50,10 @@ const ATTACH_REQUEST_TIMEOUT: Duration = Duration::from_mins(5); /// callers retry whole calls above this; past boot, exhausting this window /// means the remote boundary (or its launcher) is gone rather than still starting. const CONNECT_RETRY_TIMEOUT: Duration = Duration::from_secs(30); +/// Bound on a routed sandbox instance's first connection. The instance is +/// registered before its workload Pod is scheduled and its image pulled; the +/// driver reaps a bootstrap that is still incomplete after the same interval. +const ROUTED_FIRST_CONNECT_TIMEOUT: Duration = Duration::from_mins(5); /// Bounds the liveness check that decides whether a stream failure lost the /// whole connection. const CONNECTION_PROBE_TIMEOUT: Duration = Duration::from_secs(5); @@ -96,6 +100,8 @@ enum TransportSource { Connector(Arc), /// Accept streams the sandbox runtime opens to this supervisor. RuntimeInitiated(Arc), + /// Accept streams routed to this sandbox instance by a shared listener. + Routed(Arc), } impl BoundaryTransport { @@ -109,10 +115,34 @@ impl BoundaryTransport { | SandboxTransport::RuntimeInitiatedVsock { .. } => TransportSource::RuntimeInitiated( crate::runtime_initiated::RuntimeInitiatedListener::bind(descriptor)?, ), + SandboxTransport::RuntimeInitiatedRouted { .. } => { + return Err(BackendError::Descriptor( + "a routed runtime-initiated transport requires a shared runtime listener" + .to_string(), + )); + } _ => TransportSource::Dial, })) } + /// Accept the connections a shared listener routes to one sandbox + /// instance. The registration's handle must be the descriptor's route. + pub fn routed( + descriptor: &SandboxRuntimeDescriptor, + registration: Arc, + ) -> Result { + match &descriptor.transport { + SandboxTransport::RuntimeInitiatedRouted { route, .. } + if route == registration.handle() => + { + Ok(Self(TransportSource::Routed(registration))) + } + _ => Err(BackendError::Descriptor( + "runtime descriptor does not use this sandbox instance's route".to_string(), + )), + } + } + /// Establish raw streams through a backend-owned connector. #[must_use] pub fn connector(connector: Arc) -> Self { @@ -137,6 +167,20 @@ impl BoundaryTransport { ) })?; } + TransportSource::Routed(registration) => { + let window = if registration.has_connected() { + CONNECT_RETRY_TIMEOUT + } else { + ROUTED_FIRST_CONNECT_TIMEOUT + }; + return tokio::time::timeout(window, registration.accept(&descriptor.tls)) + .await + .map_err(|_| { + BackendError::Unavailable( + "timed out waiting for the sandbox runtime to connect".to_string(), + ) + })?; + } }; let deadline = tokio::time::Instant::now() + CONNECT_RETRY_TIMEOUT; loop { @@ -1964,7 +2008,8 @@ impl BoundaryClient { let connector_transport = transport.clone(); let authority = match &runtime_descriptor.transport { SandboxTransport::Tcp { authority, .. } - | SandboxTransport::RuntimeInitiatedTcp { authority, .. } => authority.as_str(), + | SandboxTransport::RuntimeInitiatedTcp { authority, .. } + | SandboxTransport::RuntimeInitiatedRouted { authority, .. } => authority.as_str(), SandboxTransport::Unix { .. } | SandboxTransport::Vsock { .. } | SandboxTransport::RuntimeInitiatedUnix { .. } @@ -2134,7 +2179,8 @@ async fn connect_boundary_using_connector( SandboxTransport::Vsock { guest_cid, port } => connect_host_vsock(*guest_cid, *port)?, SandboxTransport::RuntimeInitiatedTcp { .. } | SandboxTransport::RuntimeInitiatedUnix { .. } - | SandboxTransport::RuntimeInitiatedVsock { .. } => { + | SandboxTransport::RuntimeInitiatedVsock { .. } + | SandboxTransport::RuntimeInitiatedRouted { .. } => { return Err(BackendError::Descriptor( "a runtime-initiated transport cannot be dialed".to_string(), )); diff --git a/crates/openshell-sandbox-backend/src/runtime_routed.rs b/crates/openshell-sandbox-backend/src/runtime_routed.rs new file mode 100644 index 0000000000..e2da5a6742 --- /dev/null +++ b/crates/openshell-sandbox-backend/src/runtime_routed.rs @@ -0,0 +1,497 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! One supervisor listener for many sandbox instances' runtime-initiated +//! connections. +//! +//! A runtime that dials a shared supervisor first sends a fixed, bounded +//! routing preface naming an opaque route handle. The handle is only a hint: +//! it selects which sandbox instance's generation-pinned TLS identity the +//! supervisor expects, and the supervisor then completes the same pinned TLS +//! client handshake as for a dedicated listener. A peer that names another +//! instance's handle cannot present that instance's TLS identity, so it is +//! rejected before any request is carried. Every Sandbox Protocol request +//! still carries the instance's session bearer token. Unknown, removed, and +//! replaced handles are closed without a handshake, and authenticated +//! connections only fill the target instance's bounded spare queue, so no +//! peer can displace a live channel. + +use std::collections::HashMap; +use std::net::SocketAddr; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use openshell_isolation_interface::contract::{BackendError, BoundaryDuplexStream}; +use tokio::io::{AsyncRead, AsyncReadExt as _}; +use tokio::sync::{OwnedSemaphorePermit, Semaphore, mpsc}; + +use crate::boundary_protocol::SandboxTlsClientConfig; + +/// First bytes of a routing preface. +pub const ROUTE_PREFACE_MAGIC: [u8; 4] = *b"OSRP"; +/// Routing preface layout version. +pub const ROUTE_PREFACE_VERSION: u8 = 1; +/// Longest accepted route handle, in bytes. +pub const MAX_ROUTE_HANDLE_LEN: usize = 64; +const MIN_ROUTE_HANDLE_LEN: usize = 16; + +/// Unauthenticated sockets the listener handshakes at once, across sandbox +/// instances. +const MAX_PENDING_HANDSHAKES: usize = 64; +/// Unauthenticated sockets naming one handle; one handle cannot exhaust the +/// listener for every other sandbox instance. +const MAX_PENDING_HANDSHAKES_PER_ROUTE: usize = 4; +/// Bound on reading the preface and completing TLS for one socket. +const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(5); +/// Authenticated connections queued for a sandbox instance but not yet claimed. +const MAX_QUEUED_CONNECTIONS: usize = 2; + +/// Validate an opaque route handle: 16 to 64 lowercase ASCII letters, digits, +/// or hyphens. +pub fn validate_route_handle(handle: &str) -> Result<(), BackendError> { + let valid = (MIN_ROUTE_HANDLE_LEN..=MAX_ROUTE_HANDLE_LEN).contains(&handle.len()) + && handle + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-'); + if valid { + Ok(()) + } else { + Err(BackendError::Descriptor( + "runtime route handle must be 16 to 64 lowercase letters, digits, or hyphens" + .to_string(), + )) + } +} + +/// Generate a fresh opaque route handle for one runtime generation. +#[must_use] +pub fn new_route_handle() -> String { + format!("rt-{}", uuid::Uuid::new_v4().simple()) +} + +/// Encode the routing preface a runtime sends before its TLS handshake. +pub fn encode_route_preface(handle: &str) -> Result, BackendError> { + validate_route_handle(handle)?; + let length = u8::try_from(handle.len()) + .map_err(|_| BackendError::Descriptor("runtime route handle is too long".to_string()))?; + let mut preface = Vec::with_capacity(ROUTE_PREFACE_MAGIC.len() + 2 + handle.len()); + preface.extend_from_slice(&ROUTE_PREFACE_MAGIC); + preface.push(ROUTE_PREFACE_VERSION); + preface.push(length); + preface.extend_from_slice(handle.as_bytes()); + Ok(preface) +} + +/// Read and validate one routing preface. Reads only the declared bytes. +async fn read_route_preface(stream: &mut S) -> std::io::Result { + let invalid = |message: &str| std::io::Error::new(std::io::ErrorKind::InvalidData, message); + let mut header = [0_u8; 6]; + stream.read_exact(&mut header).await?; + if header[..4] != ROUTE_PREFACE_MAGIC { + return Err(invalid("missing runtime routing preface")); + } + if header[4] != ROUTE_PREFACE_VERSION { + return Err(invalid("unsupported runtime routing preface version")); + } + let length = usize::from(header[5]); + if length > MAX_ROUTE_HANDLE_LEN { + return Err(invalid("runtime route handle is too long")); + } + let mut handle = vec![0_u8; length]; + stream.read_exact(&mut handle).await?; + let handle = String::from_utf8(handle).map_err(|_| invalid("invalid runtime route handle"))?; + validate_route_handle(&handle).map_err(|error| invalid(&error.to_string()))?; + Ok(handle) +} + +struct Route { + /// Distinguishes a re-registered handle from the registration it replaced. + id: u64, + tls: SandboxTlsClientConfig, + spares: mpsc::Sender, + pending: Arc, +} + +#[derive(Default)] +struct Routes { + next_id: u64, + by_handle: HashMap, +} + +/// Shared listener that routes runtime connections to registered sandbox +/// instances. +pub struct RuntimeRouter { + routes: Arc>, + local_addr: SocketAddr, + accept_task: tokio::task::AbortHandle, +} + +impl std::fmt::Debug for RuntimeRouter { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RuntimeRouter") + .field("local_addr", &self.local_addr) + .finish_non_exhaustive() + } +} + +impl RuntimeRouter { + /// Bind `listen` and start routing authenticated runtime connections. + pub fn bind(listen: SocketAddr) -> Result, BackendError> { + let runtime = tokio::runtime::Handle::try_current().map_err(|_| { + BackendError::Unavailable("runtime router requires a Tokio runtime".into()) + })?; + let _entered = runtime.enter(); + let listener = bind_tcp(listen).map_err(|error| { + BackendError::Unavailable(format!("runtime router listener: {error}")) + })?; + let local_addr = listener.local_addr().map_err(|error| { + BackendError::Unavailable(format!("runtime router listener: {error}")) + })?; + let routes = Arc::new(Mutex::new(Routes::default())); + let accept_task = runtime + .spawn(accept_loop(listener, routes.clone())) + .abort_handle(); + tracing::info!(%local_addr, "Routed runtime-initiated Sandbox Protocol listener ready"); + Ok(Arc::new(Self { + routes, + local_addr, + accept_task, + })) + } + + #[must_use] + pub fn local_addr(&self) -> SocketAddr { + self.local_addr + } + + /// Accept connections for `handle`, authenticated with `tls`. A handle + /// that is already registered is refused; the existing instance keeps it. + pub fn register( + &self, + handle: &str, + tls: &SandboxTlsClientConfig, + ) -> Result, BackendError> { + validate_route_handle(handle)?; + crate::runtime::validate_client_tls(tls)?; + let mut routes = lock(&self.routes); + if routes.by_handle.contains_key(handle) { + return Err(BackendError::Denied( + "runtime route handle is already registered".to_string(), + )); + } + routes.next_id += 1; + let id = routes.next_id; + let (sender, spares) = mpsc::channel(MAX_QUEUED_CONNECTIONS); + routes.by_handle.insert( + handle.to_string(), + Route { + id, + tls: tls.clone(), + spares: sender, + pending: Arc::new(Semaphore::new(MAX_PENDING_HANDSHAKES_PER_ROUTE)), + }, + ); + Ok(Arc::new(RouteRegistration { + routes: Arc::downgrade(&self.routes), + handle: handle.to_string(), + id, + tls: tls.clone(), + spares: tokio::sync::Mutex::new(spares), + connected: AtomicBool::new(false), + })) + } +} + +impl Drop for RuntimeRouter { + fn drop(&mut self) { + self.accept_task.abort(); + } +} + +/// One sandbox instance's claim on a route handle. Dropping it unregisters the handle +/// and closes connections queued for it. +pub struct RouteRegistration { + routes: std::sync::Weak>, + handle: String, + id: u64, + tls: SandboxTlsClientConfig, + spares: tokio::sync::Mutex>, + /// The runtime has delivered at least one authenticated connection. + connected: AtomicBool, +} + +impl std::fmt::Debug for RouteRegistration { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RouteRegistration") + .field("handle", &self.handle) + .finish_non_exhaustive() + } +} + +impl RouteRegistration { + #[must_use] + pub fn handle(&self) -> &str { + &self.handle + } + + /// Wait for one authenticated connection for `descriptor`'s sandbox + /// instance. + pub(crate) async fn accept( + &self, + tls: &SandboxTlsClientConfig, + ) -> Result { + if *tls != self.tls { + return Err(BackendError::Descriptor( + "runtime route is registered for a different sandbox instance".to_string(), + )); + } + let stream = self + .spares + .lock() + .await + .recv() + .await + .ok_or_else(|| BackendError::Unavailable("runtime router stopped".to_string()))?; + self.connected.store(true, Ordering::Relaxed); + Ok(stream) + } + + /// Whether the runtime has connected for this sandbox instance before. + pub(crate) fn has_connected(&self) -> bool { + self.connected.load(Ordering::Relaxed) + } +} + +impl Drop for RouteRegistration { + fn drop(&mut self) { + if let Some(routes) = self.routes.upgrade() { + let mut routes = lock(&routes); + if routes + .by_handle + .get(&self.handle) + .is_some_and(|route| route.id == self.id) + { + routes.by_handle.remove(&self.handle); + } + } + } +} + +fn lock(routes: &Mutex) -> std::sync::MutexGuard<'_, Routes> { + routes + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +fn bind_tcp(address: SocketAddr) -> std::io::Result { + let socket = socket2::Socket::new( + socket2::Domain::for_address(address), + socket2::Type::STREAM, + Some(socket2::Protocol::TCP), + )?; + socket.set_reuse_address(true)?; + socket.set_nonblocking(true)?; + socket.bind(&address.into())?; + socket.listen(128)?; + tokio::net::TcpListener::from_std(socket.into()) +} + +/// Route each accepted socket within global and per-handle handshake bounds. +async fn accept_loop(listener: tokio::net::TcpListener, routes: Arc>) { + let pending = Arc::new(Semaphore::new(MAX_PENDING_HANDSHAKES)); + // Owned here so aborting the listener also cancels pending handshakes. + let mut handshakes = tokio::task::JoinSet::new(); + loop { + tokio::select! { + accepted = listener.accept() => { + let stream = match accepted { + Ok((stream, _)) => stream, + Err(error) => { + // Descriptor exhaustion and similar errors are transient. + tracing::warn!(%error, "routed runtime accept failed"); + tokio::time::sleep(Duration::from_millis(100)).await; + continue; + } + }; + let Ok(permit) = pending.clone().try_acquire_owned() else { + tracing::debug!("routed runtime handshake limit reached; closing connection"); + continue; + }; + openshell_core::net::set_tcp_nodelay_best_effort(&stream); + crate::runtime::enable_boundary_tcp_keepalive(&stream); + let routes = routes.clone(); + handshakes.spawn(async move { + if let Err(error) = route_connection(stream, &routes, permit).await { + tracing::debug!(%error, "rejected routed runtime connection"); + } + }); + } + Some(_) = handshakes.join_next(), if !handshakes.is_empty() => {} + } + } +} + +async fn route_connection( + mut stream: tokio::net::TcpStream, + routes: &Mutex, + _listener_permit: OwnedSemaphorePermit, +) -> Result<(), String> { + let deadline = tokio::time::Instant::now() + HANDSHAKE_TIMEOUT; + let handle = tokio::time::timeout_at(deadline, read_route_preface(&mut stream)) + .await + .map_err(|_| "routing preface timed out".to_string())? + .map_err(|error| format!("routing preface: {error}"))?; + // The handle only selects which pinned identity to expect. + let (tls, spares, route_pending) = { + let routes = lock(routes); + let route = routes + .by_handle + .get(&handle) + .ok_or_else(|| "unknown or retired runtime route handle".to_string())?; + ( + route.tls.clone(), + route.spares.clone(), + route.pending.clone(), + ) + }; + let Ok(_route_permit) = route_pending.try_acquire_owned() else { + return Err("runtime route handshake limit reached".to_string()); + }; + let stream: BoundaryDuplexStream = Box::new(stream); + let authenticated = tokio::time::timeout_at( + deadline, + crate::runtime::authenticate_boundary_stream(&tls, stream), + ) + .await + .map_err(|_| "runtime TLS handshake timed out".to_string())? + .map_err(|error| error.to_string())?; + // A sender captured before its registration was removed or replaced has + // no receiver, so the stream cannot reach another generation. + spares + .try_send(authenticated) + .map_err(|_| "runtime route has no free spare slot".to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::boundary_protocol::{SandboxTlsMaterial, generate_sandbox_tls_material}; + use tokio::io::AsyncWriteExt as _; + + /// A runtime: dial the router, name `route`, and serve `identity`'s TLS. + async fn runtime_dial( + router: SocketAddr, + route: &str, + identity: &SandboxTlsMaterial, + ) -> std::io::Result> { + let mut stream = tokio::net::TcpStream::connect(router).await?; + stream + .write_all(&encode_route_preface(route).unwrap()) + .await?; + let mut config = rustls::ServerConfig::builder_with_provider( + openshell_crypto::tls::configuration_provider(), + ) + .with_protocol_versions(&[&rustls::version::TLS13]) + .unwrap() + .with_no_client_auth() + .with_single_cert( + rustls_pemfile::certs(&mut identity.certificate_chain_pem.as_bytes()) + .collect::, _>>() + .unwrap(), + rustls_pemfile::private_key(&mut identity.private_key_pem.as_bytes()) + .unwrap() + .unwrap(), + ) + .unwrap(); + config.alpn_protocols = vec![b"h2".to_vec()]; + tokio_rustls::TlsAcceptor::from(Arc::new(config)) + .accept(stream) + .await + } + + fn client_tls(identity: &SandboxTlsMaterial) -> SandboxTlsClientConfig { + SandboxTlsClientConfig { + server_name: identity.server_name.clone(), + trust_anchor_pem: identity.trust_anchor_pem.clone(), + } + } + + async fn delivered(registration: &RouteRegistration, tls: &SandboxTlsClientConfig) -> bool { + tokio::time::timeout(Duration::from_millis(300), registration.accept(tls)) + .await + .is_ok_and(|accepted| accepted.is_ok()) + } + + #[tokio::test] + async fn routes_only_authenticated_runtimes_to_their_own_live_instance() { + let router = RuntimeRouter::bind("127.0.0.1:0".parse().unwrap()).unwrap(); + let address = router.local_addr(); + let alpha = generate_sandbox_tls_material(openshell_core::SandboxSessionId::new()).unwrap(); + let beta = generate_sandbox_tls_material(openshell_core::SandboxSessionId::new()).unwrap(); + let (alpha_route, beta_route) = (new_route_handle(), new_route_handle()); + let alpha_registration = router.register(&alpha_route, &client_tls(&alpha)).unwrap(); + let beta_registration = router.register(&beta_route, &client_tls(&beta)).unwrap(); + assert!( + router.register(&beta_route, &client_tls(&alpha)).is_err(), + "a live route cannot be claimed again" + ); + + // A runtime holding the named instance's identity is delivered to it. + let alpha_tls = client_tls(&alpha); + let (runtime, accepted) = tokio::join!( + runtime_dial(address, &alpha_route, &alpha), + alpha_registration.accept(&alpha_tls) + ); + assert!(runtime.is_ok() && accepted.is_ok()); + + // Naming another instance without its identity fails TLS, and + // nothing reaches either instance. + assert!(runtime_dial(address, &beta_route, &alpha).await.is_err()); + assert!(!delivered(&beta_registration, &client_tls(&beta)).await); + assert!(!delivered(&alpha_registration, &client_tls(&alpha)).await); + + // Unknown and retired routes are closed before any TLS handshake. + assert!( + runtime_dial(address, &new_route_handle(), &beta) + .await + .is_err() + ); + drop(alpha_registration); + assert!(runtime_dial(address, &alpha_route, &alpha).await.is_err()); + + // The other instance is unaffected and keeps accepting its runtime. + let beta_tls = client_tls(&beta); + let (runtime, accepted) = tokio::join!( + runtime_dial(address, &beta_route, &beta), + beta_registration.accept(&beta_tls) + ); + assert!(runtime.is_ok() && accepted.is_ok()); + + // A retired route can be registered again by a new owner. + assert!(router.register(&alpha_route, &client_tls(&alpha)).is_ok()); + } + + #[tokio::test] + async fn preface_round_trips_and_rejects_malformed_input() { + let handle = new_route_handle(); + let encoded = encode_route_preface(&handle).unwrap(); + assert_eq!( + read_route_preface(&mut encoded.as_slice()).await.unwrap(), + handle + ); + let mut wrong_version = encoded.clone(); + wrong_version[4] = 2; + for malformed in [ + b"TLS\x16\x03\x01".to_vec(), + wrong_version, + [b"OSRP\x01\x41".as_slice(), &[b'a'; 65]].concat(), + [b"OSRP\x01\x10".as_slice(), b"UPPERCASE-HANDLE"].concat(), + ] { + assert!(read_route_preface(&mut malformed.as_slice()).await.is_err()); + } + assert!(encode_route_preface("short").is_err()); + } +} diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index 578522e4c3..56a974b9a1 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -382,7 +382,14 @@ mod linux { authority, addresses, tls, - } => validate_runtime_initiated_targets(authority, addresses, tls)?, + route, + } => { + validate_runtime_initiated_targets(authority, addresses, tls)?; + if let Some(route) = route { + openshell_sandbox_backend::runtime_routed::validate_route_handle(route) + .map_err(|error| error.to_string())?; + } + } BoundaryListenerConfig::Unix { .. } | BoundaryListenerConfig::RuntimeInitiatedUnix { .. } | BoundaryListenerConfig::RuntimeInitiatedVsock { .. } @@ -742,16 +749,30 @@ mod linux { } /// Dial only driver-provided supervisor addresses from the bootstrap. + /// + /// A shared supervisor first receives this generation's route handle so + /// it can expect the right pinned identity; this runtime then serves TLS + /// exactly as it does for a dedicated supervisor. async fn dial_supervisor(config: &BoundaryListenerConfig) -> io::Result { match config { - BoundaryListenerConfig::RuntimeInitiatedTlsTcp { addresses, .. } => { - let stream = + BoundaryListenerConfig::RuntimeInitiatedTlsTcp { + addresses, route, .. + } => { + let mut stream = openshell_core::net::connect_tcp_nodelay_best_effort(addresses).await?; let keepalive = socket2::TcpKeepalive::new() .with_time(Duration::from_secs(10)) .with_interval(Duration::from_secs(5)) .with_retries(3); let _ = socket2::SockRef::from(&stream).set_tcp_keepalive(&keepalive); + if let Some(route) = route { + let preface = + openshell_sandbox_backend::runtime_routed::encode_route_preface(route) + .map_err(|error| { + io::Error::new(io::ErrorKind::InvalidInput, error.to_string()) + })?; + stream.write_all(&preface).await?; + } Ok(PlainControlStream::Tcp(stream.into_std()?)) } BoundaryListenerConfig::RuntimeInitiatedUnix { socket_path, .. } => { diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index 0066cd8d5b..9160719bf9 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -25,6 +25,7 @@ openshell-policy = { path = "../openshell-policy" } openshell-prover = { path = "../openshell-prover" } openshell-providers = { path = "../openshell-providers" } openshell-supervisor-middleware = { path = "../openshell-supervisor-middleware" } +openshell-supervisor-control = { path = "../openshell-supervisor-control" } openshell-supervisor-middleware-builtins = { path = "../openshell-supervisor-middleware-builtins" } # Kubernetes client used by ServiceAccount bootstrap authentication and the diff --git a/crates/openshell-server/src/auth/audit.rs b/crates/openshell-server/src/auth/audit.rs index 4d4248ceb1..6c9136aa6f 100644 --- a/crates/openshell-server/src/auth/audit.rs +++ b/crates/openshell-server/src/auth/audit.rs @@ -216,7 +216,10 @@ fn authenticated(principal: &Principal) -> bool { match principal { Principal::User(user) => user.identity.provider != IdentityProvider::LocalDev, Principal::Anonymous => false, - Principal::Operator(_) | Principal::Sandbox(_) | Principal::Peer(_) => true, + Principal::Operator(_) + | Principal::Sandbox(_) + | Principal::Peer(_) + | Principal::Supervisor(_) => true, } } @@ -234,6 +237,7 @@ fn mechanism(principal: &Principal) -> Option { SandboxIdentitySource::ComputeDriver { .. } => AuthMechanism::ComputeDriver, }), Principal::Peer(_) => Some(AuthMechanism::GatewayPeer), + Principal::Supervisor(_) => Some(AuthMechanism::SupervisorToken), Principal::Anonymous => None, } } @@ -506,6 +510,9 @@ fn principal_user(principal: &Principal) -> User { Principal::Operator(operator) => User::new(&operator.identity.subject, UserTypeId::Admin), Principal::Sandbox(sandbox) => User::other(&sandbox.sandbox_id, "Sandbox"), Principal::Peer(peer) => User::other(&peer.replica_id, "Gateway Peer"), + Principal::Supervisor(supervisor) => { + User::other(&supervisor.identity.subject, "Supervisor") + } Principal::Anonymous => User::unidentified(), } } diff --git a/crates/openshell-server/src/auth/authenticator.rs b/crates/openshell-server/src/auth/authenticator.rs index db789213fa..c9fc367a3f 100644 --- a/crates/openshell-server/src/auth/authenticator.rs +++ b/crates/openshell-server/src/auth/authenticator.rs @@ -19,6 +19,8 @@ //! gateway session JWTs //! - [`super::compute_driver::ComputeDriverAuthenticator`] — driver-native //! sandbox bootstrap credentials (path-scoped to `IssueSandboxToken`) +//! - [`super::supervisor_token::SupervisorTokenAuthenticator`] — the shared +//! supervisor's gateway-minted token for the assignment stream //! - [`super::oidc::OidcAuthenticator`] — user OIDC Bearer tokens use super::principal::Principal; use async_trait::async_trait; @@ -118,6 +120,8 @@ pub enum AuthMechanism { GatewayPeer, /// The `CreateSshSession` token an SSH relay presents. SshSessionToken, + /// The shared supervisor's gateway-minted supervisor token. + SupervisorToken, } impl AuthMechanism { @@ -131,6 +135,7 @@ impl AuthMechanism { Self::ComputeDriver => "compute_driver", Self::GatewayPeer => "k8s_service_account", Self::SshSessionToken => "ssh_session_token", + Self::SupervisorToken => "supervisor_token", } } } diff --git a/crates/openshell-server/src/auth/descriptor_authz.rs b/crates/openshell-server/src/auth/descriptor_authz.rs index cbdadbdb29..13e669a27b 100644 --- a/crates/openshell-server/src/auth/descriptor_authz.rs +++ b/crates/openshell-server/src/auth/descriptor_authz.rs @@ -17,7 +17,7 @@ use super::method_authz::{AuthMode, Role}; const AUTHORIZATION_EXTENSION: &str = "openshell.options.v1.authorization"; /// Gateway-served protobuf packages. -const GATEWAY_PACKAGES: &[&str] = &["openshell.v1"]; +const GATEWAY_PACKAGES: &[&str] = &["openshell.v1", "openshell.supervisor.host.v1"]; /// Bearer-authenticated methods that deliberately require no role or scope. /// @@ -125,6 +125,7 @@ impl DescriptorAuthTable { "dual" => AuthMode::Dual, "peer" => AuthMode::Peer, "operator" => AuthMode::Operator, + "supervisor" => AuthMode::Supervisor, other => { return Err(format!("method {path}: unknown auth_mode '{other}'")); } diff --git a/crates/openshell-server/src/auth/guard.rs b/crates/openshell-server/src/auth/guard.rs index 2218932942..2578ec44be 100644 --- a/crates/openshell-server/src/auth/guard.rs +++ b/crates/openshell-server/src/auth/guard.rs @@ -30,10 +30,10 @@ use tonic::Status; pub fn ensure_sandbox_scope(principal: &Principal, claimed_sandbox_id: &str) -> Result<(), Status> { match principal { Principal::User(_) | Principal::Operator(_) => Ok(()), - Principal::Peer(_) => Err(deny( + Principal::Peer(_) | Principal::Supervisor(_) => Err(deny( principal, AuthzDenial::MethodNotPermitted, - "gateway peer principals may not call sandbox-scoped methods", + "gateway peer and supervisor principals may not call sandbox-scoped methods", )), Principal::Sandbox(p) => { if p.sandbox_id == claimed_sandbox_id { @@ -88,7 +88,10 @@ pub fn ensure_sandbox_principal_scope( ensure_sandbox_scope(principal, claimed_sandbox_id)?; Ok(p.clone()) } - Principal::User(_) | Principal::Operator(_) | Principal::Peer(_) => Err(deny( + Principal::User(_) + | Principal::Operator(_) + | Principal::Peer(_) + | Principal::Supervisor(_) => Err(deny( principal, AuthzDenial::MethodNotPermitted, "supervisor RPCs require a sandbox principal", diff --git a/crates/openshell-server/src/auth/launch_signing.rs b/crates/openshell-server/src/auth/launch_signing.rs index a1b510bb24..c6eebcdd69 100644 --- a/crates/openshell-server/src/auth/launch_signing.rs +++ b/crates/openshell-server/src/auth/launch_signing.rs @@ -11,6 +11,7 @@ use super::sandbox_jwt::{ExtensionJwtIssuer, SandboxSessionJwtAuthority}; pub struct LaunchSigningAuthorities { pub extension: Arc, + /// Also mints and verifies the shared supervisor's token with the same key. pub sandbox_session: Arc, } diff --git a/crates/openshell-server/src/auth/method_authz.rs b/crates/openshell-server/src/auth/method_authz.rs index 1efefb3564..e00357b8ac 100644 --- a/crates/openshell-server/src/auth/method_authz.rs +++ b/crates/openshell-server/src/auth/method_authz.rs @@ -26,6 +26,9 @@ pub enum AuthMode { Peer, /// Only a verified mTLS operator may call this method. Operator, + /// Only the gateway's shared supervisor, authenticated by its + /// gateway-minted supervisor token, may call this method. + Supervisor, } /// Coarse role mapping. Maps to the configured `admin_role` / @@ -91,7 +94,11 @@ pub fn is_sandbox_callable(method: &str) -> bool { pub fn is_user_callable(method: &str) -> bool { match lookup(method).map(|m| m.auth_mode) { Some( - AuthMode::Sandbox | AuthMode::Unauthenticated | AuthMode::Peer | AuthMode::Operator, + AuthMode::Sandbox + | AuthMode::Unauthenticated + | AuthMode::Peer + | AuthMode::Operator + | AuthMode::Supervisor, ) => false, Some(AuthMode::Bearer | AuthMode::Dual) | None => true, } @@ -112,6 +119,16 @@ pub fn is_peer_callable(method: &str) -> bool { matches!(lookup(method).map(|m| m.auth_mode), Some(AuthMode::Peer)) } +/// `true` if the method is callable by the shared supervisor, and by nothing +/// else. +#[must_use] +pub fn is_supervisor_callable(method: &str) -> bool { + matches!( + lookup(method).map(|m| m.auth_mode), + Some(AuthMode::Supervisor) + ) +} + #[cfg(test)] mod tests { use super::*; @@ -173,6 +190,17 @@ mod tests { assert!(!is_user_callable(path)); assert!(is_peer_callable(path)); } + // The supervisor assignment stream is reachable only with a + // supervisor token. + let host_attach = "/openshell.supervisor.host.v1.SupervisorHost/Attach"; + assert!(is_supervisor_callable(host_attach)); + assert!(!is_user_callable(host_attach)); + assert!(!is_operator_callable(host_attach)); + assert!(!is_sandbox_callable(host_attach)); + assert!(!is_peer_callable(host_attach)); + assert!(!is_supervisor_callable( + "/openshell.v1.OpenShell/ConnectSupervisor" + )); // Unauthenticated methods are not "user callable" — they're // intercepted before principal evaluation. assert!(!is_user_callable("/openshell.v1.OpenShell/Health")); diff --git a/crates/openshell-server/src/auth/mod.rs b/crates/openshell-server/src/auth/mod.rs index 5b0209cae4..f1800d5044 100644 --- a/crates/openshell-server/src/auth/mod.rs +++ b/crates/openshell-server/src/auth/mod.rs @@ -25,6 +25,7 @@ pub mod principal; pub mod sandbox_jwt; pub mod sandbox_methods; pub mod sandbox_session; +pub mod supervisor_token; pub mod workspace_authz; pub use http::router; diff --git a/crates/openshell-server/src/auth/principal.rs b/crates/openshell-server/src/auth/principal.rs index e9809133d1..bc5d31381a 100644 --- a/crates/openshell-server/src/auth/principal.rs +++ b/crates/openshell-server/src/auth/principal.rs @@ -32,6 +32,10 @@ pub enum Principal { Sandbox(#[allow(dead_code)] SandboxPrincipal), /// Gateway replica authenticated for internal peer RPCs. Peer(PeerPrincipal), + /// The gateway's shared supervisor, authenticated by a gateway-minted + /// supervisor token. It may call only the supervisor assignment stream + /// and never acts for a user or a sandbox. + Supervisor(SupervisorPrincipal), /// Truly unauthenticated caller (health probes, reflection). Sandbox-class /// and user-class methods reject this variant. #[allow(dead_code)] @@ -81,6 +85,13 @@ pub struct SandboxPrincipal { pub trust_domain: Option, } +/// Shared supervisor caller. +#[derive(Debug, Clone)] +pub struct SupervisorPrincipal { + /// Claims of the verified supervisor token. + pub identity: openshell_supervisor_control::HostIdentity, +} + /// Gateway peer caller. #[derive(Debug, Clone)] pub struct PeerPrincipal { @@ -119,6 +130,7 @@ impl Principal { Self::Operator(operator) => format!("operator:{}", operator.identity.subject), Self::Sandbox(sandbox) => format!("sandbox:{}", sandbox.sandbox_id), Self::Peer(peer) => format!("peer:{}", peer.replica_id), + Self::Supervisor(supervisor) => format!("supervisor:{}", supervisor.identity.subject), Self::Anonymous => "anonymous".to_string(), } } diff --git a/crates/openshell-server/src/auth/sandbox_jwt.rs b/crates/openshell-server/src/auth/sandbox_jwt.rs index 57c64385f5..774b9f5f47 100644 --- a/crates/openshell-server/src/auth/sandbox_jwt.rs +++ b/crates/openshell-server/src/auth/sandbox_jwt.rs @@ -4,9 +4,11 @@ //! Gateway-minted session and extension JWTs. //! //! [`SandboxSessionJwtAuthority`] mints and verifies generation-bound sandbox -//! session tokens. [`ExtensionJwtIssuer`] mints explicitly typed credentials -//! for supervisor middleware and gateway interceptors and publishes the -//! corresponding JWKS metadata. +//! session tokens, and with the same key the shared supervisor's token for +//! the private assignment stream (its own `typ` and claims, see +//! [`super::supervisor_token`]). [`ExtensionJwtIssuer`] mints explicitly +//! typed credentials for supervisor middleware and gateway interceptors and +//! publishes the corresponding JWKS metadata. //! //! Algorithm: `EdDSA` (Ed25519). Pinned via `Validation::algorithms` to //! prevent algorithm-confusion attacks. @@ -35,9 +37,10 @@ use x509_parser::{oid_registry::OID_SIG_ED25519, prelude::FromDer, x509::Subject use openshell_core::SandboxSessionId; use openshell_core::jwt::{ - AuthenticatedSandboxSession, CredentialEpoch, GATEWAY_SESSION_JWT_TYPE, SandboxId, - SandboxLaunchAuthentication, SandboxRuntimeIdentity, SessionJwtError, SessionJwtIssuer, - SessionJwtVerifier, SessionTokenProfile, SessionVerificationKey, SupervisorAuthBundle, + AuthenticatedSandboxSession, AuthenticatedSupervisor, CredentialEpoch, + GATEWAY_SESSION_JWT_TYPE, MintedSessionToken, SandboxId, SandboxLaunchAuthentication, + SandboxRuntimeIdentity, SessionJwtError, SessionJwtIssuer, SessionJwtVerifier, + SessionTokenProfile, SessionVerificationKey, SupervisorAuthBundle, SupervisorJwtVerifier, SystemJwtClock, }; use openshell_core::sandbox_generation::SandboxGenerationId; @@ -102,6 +105,7 @@ pub struct MintedToken { pub struct SandboxSessionJwtAuthority { issuer: SessionJwtIssuer, gateway_verifier: SessionJwtVerifier, + supervisor_verifier: SupervisorJwtVerifier, gateway_id: String, verification_keys: Vec, } @@ -148,12 +152,16 @@ impl SandboxSessionJwtAuthority { gateway_id, SessionTokenProfile::Gateway, verification_keys.clone(), - clock, + clock.clone(), ) .map_err(|error| error.to_string())?; + let supervisor_verifier = + SupervisorJwtVerifier::new(gateway_id, verification_keys.clone(), clock) + .map_err(|error| error.to_string())?; Ok(Self { issuer, gateway_verifier, + supervisor_verifier, gateway_id: gateway_id.to_string(), verification_keys, }) @@ -276,6 +284,34 @@ impl SandboxSessionJwtAuthority { ) }) } + + /// Mint the shared supervisor's token for the assignment stream. Its + /// lifetime is bounded on its own; the session lifetime may be + /// non-expiring. + pub fn mint_supervisor_token( + &self, + ttl: Duration, + ) -> Result { + self.issuer.mint_supervisor_token(ttl) + } + + /// Verify a presented supervisor token. + pub fn verify_supervisor_token( + &self, + token: &str, + ) -> Result { + self.supervisor_verifier.verify(token).map_err(|error| { + let reason = match error { + SessionJwtError::SubjectMismatch => AuthFailureReason::AudienceOrIssuerMismatch, + ref other => session_failure_reason(other), + }; + AuthRejection::new( + AuthMechanism::SupervisorToken, + reason, + Status::unauthenticated(format!("invalid supervisor token: {error}")), + ) + }) + } } /// Authenticates launch-scoped supervisor tokens and checks their identity diff --git a/crates/openshell-server/src/auth/supervisor_token.rs b/crates/openshell-server/src/auth/supervisor_token.rs new file mode 100644 index 0000000000..b8c77ec233 --- /dev/null +++ b/crates/openshell-server/src/auth/supervisor_token.rs @@ -0,0 +1,187 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The supervisor token: the credential of the gateway's shared supervisor +//! for the private assignment stream. +//! +//! The token is minted and verified by the launch-signing +//! [`super::sandbox_jwt::SandboxSessionJwtAuthority`] with the same Ed25519 +//! key, key ID, and gateway identity as the sandbox session tokens, under a +//! distinct profile (`typ`, subject, claims), so no sandbox, extension, user, +//! or peer credential is accepted on the stream and the supervisor token is +//! accepted nowhere else. [`SupervisorTokenAuthenticator`] slots the verifier +//! into the gateway's authenticator chain. + +use std::sync::Arc; + +use async_trait::async_trait; +use jsonwebtoken::decode_header; +use openshell_core::jwt::SUPERVISOR_JWT_TYPE; +use openshell_supervisor_control::HostIdentity; + +use super::authenticator::{AuthRejection, Authentication, Authenticator, bearer_credential}; +use super::principal::{Principal, SupervisorPrincipal}; +use super::sandbox_jwt::SandboxSessionJwtAuthority; + +/// Authenticates bearers typed as supervisor tokens. Any other bearer is not +/// this authenticator's business; a supervisor-typed bearer that fails +/// verification ends the chain. +pub struct SupervisorTokenAuthenticator { + authority: Arc, +} + +impl SupervisorTokenAuthenticator { + pub fn new(authority: Arc) -> Self { + Self { authority } + } +} + +impl std::fmt::Debug for SupervisorTokenAuthenticator { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("SupervisorTokenAuthenticator") + .finish_non_exhaustive() + } +} + +#[async_trait] +impl Authenticator for SupervisorTokenAuthenticator { + async fn authenticate( + &self, + headers: &http::HeaderMap, + _path: &str, + ) -> Result { + let Some(token) = bearer_credential(headers) else { + return Ok(Authentication::NotApplicable); + }; + let Ok(header) = decode_header(token) else { + return Ok(Authentication::NotApplicable); + }; + if header.typ.as_deref() != Some(SUPERVISOR_JWT_TYPE) { + return Ok(Authentication::NotApplicable); + } + let authenticated = self.authority.verify_supervisor_token(token)?; + Ok(Authentication::Accepted(Principal::Supervisor( + SupervisorPrincipal { + identity: HostIdentity { + subject: authenticated.subject, + token_id: authenticated.token_id.to_string(), + expires_at: authenticated.expires_at, + }, + }, + ))) + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + use crate::auth::authenticator::{AuthFailureReason, AuthMechanism}; + use openshell_bootstrap::jwt::generate_jwt_key; + + fn authority(gateway_id: &str) -> Arc { + let material = generate_jwt_key().expect("jwt key"); + Arc::new( + SandboxSessionJwtAuthority::from_pem( + material.signing_key_pem.as_bytes(), + material.public_key_pem.as_bytes(), + material.kid, + gateway_id, + None, + ) + .expect("authority"), + ) + } + + fn headers(token: &str) -> http::HeaderMap { + let mut headers = http::HeaderMap::new(); + headers.insert( + http::header::AUTHORIZATION, + format!("Bearer {token}").parse().unwrap(), + ); + headers + } + + #[tokio::test] + async fn supervisor_tokens_authenticate_only_as_the_supervisor() { + let attach = "/openshell.supervisor.host.v1.SupervisorHost/Attach"; + let gateway = authority("test-gateway"); + let minted = gateway + .mint_supervisor_token(Duration::from_mins(15)) + .expect("mint"); + let authenticator = SupervisorTokenAuthenticator::new(gateway.clone()); + + let accepted = authenticator + .authenticate(&headers(minted.token.expose_secret()), attach) + .await + .expect("valid token"); + match accepted { + Authentication::Accepted(Principal::Supervisor(supervisor)) => { + assert_eq!( + supervisor.identity.subject, + "openshell-supervisor:test-gateway" + ); + assert_eq!(supervisor.identity.token_id, minted.token_id.to_string()); + assert_eq!(supervisor.identity.expires_at, minted.expires_at); + } + other => panic!("unexpected outcome {other:?}"), + } + + // Session tokens of the same authority, opaque bearers, and non-JWT + // bearers fall through to the other authenticators. + let identity = crate::auth::sandbox_session::PersistedSandboxIdentity::new().unwrap(); + let launch = gateway + .mint_persisted_launch("sandbox-a", &identity) + .unwrap(); + for bearer in [ + launch.supervisor.gateway_token.expose_secret(), + "opaque-user-token", + "a.b.c", + ] { + assert!(matches!( + authenticator + .authenticate(&headers(bearer), attach) + .await + .unwrap(), + Authentication::NotApplicable + )); + } + // And the session verifier refuses the supervisor token. + assert!( + gateway + .verify_gateway_token(minted.token.expose_secret()) + .is_err() + ); + + // A supervisor-typed token signed by another gateway's key fails + // closed, as does one for another gateway identity. + let other_key = authority("test-gateway"); + let foreign = other_key + .mint_supervisor_token(Duration::from_mins(15)) + .unwrap(); + let rejection = authenticator + .authenticate(&headers(foreign.token.expose_secret()), attach) + .await + .expect_err("foreign key must be rejected"); + assert_eq!(rejection.mechanism, AuthMechanism::SupervisorToken); + assert_eq!(rejection.reason, AuthFailureReason::InvalidCredential); + assert_eq!(rejection.status.code(), tonic::Code::Unauthenticated); + let other_gateway = authority("other-gateway"); + let rejection = SupervisorTokenAuthenticator::new(other_gateway.clone()) + .authenticate( + &headers( + other_gateway + .mint_supervisor_token(Duration::from_mins(15)) + .unwrap() + .token + .expose_secret(), + ), + attach, + ) + .await + .expect("own token verifies"); + assert!(matches!(rejection, Authentication::Accepted(_))); + } +} diff --git a/crates/openshell-server/src/auth/workspace_authz.rs b/crates/openshell-server/src/auth/workspace_authz.rs index d7c746ada6..bc5cac8778 100644 --- a/crates/openshell-server/src/auth/workspace_authz.rs +++ b/crates/openshell-server/src/auth/workspace_authz.rs @@ -207,10 +207,10 @@ pub async fn authorize_workspace( workspace, grant: AuthGrant::Sandbox, }), - Principal::Peer(_) => Err(deny( + Principal::Peer(_) | Principal::Supervisor(_) => Err(deny( principal, AuthzDenial::MethodNotPermitted, - "gateway peer principals cannot perform workspace operations", + "gateway peer and supervisor principals cannot perform workspace operations", )), Principal::Anonymous => Err(Status::unauthenticated("authentication required")), } @@ -244,7 +244,9 @@ pub fn require_platform_admin(admin_role: &str, principal: &Principal) -> Result } Principal::User(_) => "platform admin role required for cross-workspace operations", Principal::Sandbox(_) => "sandbox principals cannot perform cross-workspace operations", - Principal::Peer(_) => "gateway peer principals cannot perform cross-workspace operations", + Principal::Peer(_) | Principal::Supervisor(_) => { + "gateway peer and supervisor principals cannot perform cross-workspace operations" + } Principal::Anonymous => return Err(Status::unauthenticated("authentication required")), }; Err(deny(principal, AuthzDenial::PlatformAdminRequired, message)) diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index a100aa11d1..7c31f30cfe 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -763,6 +763,9 @@ pub struct ComputeRuntime { restart_notify: Arc, ssh_identities: Arc>, supervisor_ocsf_headers: Arc>, + /// Desired assignments of the shared supervisor host, when the gateway + /// coordinates one for the in-process driver. + supervisor_host: Option, } impl fmt::Debug for ComputeRuntime { @@ -871,9 +874,28 @@ impl ComputeRuntime { restart_notify: Arc::new(Notify::new()), ssh_identities: Arc::new(OnceLock::new()), supervisor_ocsf_headers: Arc::new(OnceLock::new()), + supervisor_host: None, }) } + /// Serve the shared supervisor host the gateway coordinates for the + /// in-process driver. + pub(crate) fn with_supervisor_host( + mut self, + supervisor_host: Option, + ) -> Self { + self.supervisor_host = supervisor_host; + self + } + + /// Gateway-owned assignments of the shared supervisor host, if any. + #[must_use] + pub fn supervisor_host( + &self, + ) -> Option<&openshell_supervisor_control::SupervisorHostController> { + self.supervisor_host.as_ref() + } + pub(crate) fn configure_ssh_identities( &self, credentials: crate::credentials::CredentialRuntime, @@ -7850,6 +7872,7 @@ pub fn new_test_runtime_with_driver( restart_notify: Arc::new(Notify::new()), ssh_identities: Arc::new(OnceLock::new()), supervisor_ocsf_headers: Arc::new(OnceLock::new()), + supervisor_host: None, } } @@ -8970,6 +8993,7 @@ mod tests { restart_notify: Arc::new(Notify::new()), ssh_identities: Arc::new(OnceLock::new()), supervisor_ocsf_headers: Arc::new(OnceLock::new()), + supervisor_host: None, } } diff --git a/crates/openshell-server/src/grpc/mod.rs b/crates/openshell-server/src/grpc/mod.rs index 35f16c0441..0368025348 100644 --- a/crates/openshell-server/src/grpc/mod.rs +++ b/crates/openshell-server/src/grpc/mod.rs @@ -10,6 +10,7 @@ pub mod provider; mod provider_credentials; pub mod provider_readiness; mod sandbox; +pub mod supervisor_host; pub use sandbox::mint_persisted_authentication; mod service; mod validation; diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 0e3dc96a24..374a3b5759 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -2788,8 +2788,8 @@ async fn resolve_sandbox_by_name_for_principal( Principal::User(_) | Principal::Operator(_) => { sandbox.ok_or_else(|| Status::not_found("sandbox not found")) } - Principal::Peer(_) => Err(Status::permission_denied( - "gateway peer principals may not resolve sandbox configuration", + Principal::Peer(_) | Principal::Supervisor(_) => Err(Status::permission_denied( + "gateway peer and supervisor principals may not resolve sandbox configuration", )), Principal::Anonymous => Err(Status::unauthenticated( "sandbox-scoped methods require an authenticated caller", diff --git a/crates/openshell-server/src/grpc/provider.rs b/crates/openshell-server/src/grpc/provider.rs index 8abcbe037e..b77b9ed45b 100644 --- a/crates/openshell-server/src/grpc/provider.rs +++ b/crates/openshell-server/src/grpc/provider.rs @@ -4293,7 +4293,8 @@ async fn exchange_provider_subject_token_inner( Principal::Sandbox(_) | Principal::User(_) | Principal::Operator(_) - | Principal::Peer(_), + | Principal::Peer(_) + | Principal::Supervisor(_), ) => Some(tonic::Code::PermissionDenied), Some(Principal::Anonymous) | None => Some(tonic::Code::Unauthenticated), }; diff --git a/crates/openshell-server/src/grpc/supervisor_host.rs b/crates/openshell-server/src/grpc/supervisor_host.rs new file mode 100644 index 0000000000..e51ca87097 --- /dev/null +++ b/crates/openshell-server/src/grpc/supervisor_host.rs @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The private `openshell.supervisor.host.v1` service: the gateway end of the +//! shared supervisor's assignment stream. +//! +//! The router admits only requests bearing a verified supervisor token, so +//! the handler sees a [`Principal::Supervisor`] and hands the stream to the +//! controller the in-process compute driver installed. + +use std::pin::Pin; +use std::sync::Arc; + +use openshell_core::proto::supervisor_host::v1::supervisor_host_server::SupervisorHost; +use openshell_core::proto::supervisor_host::v1::{HostGatewayMessage, HostMessage}; +use tonic::{Request, Response, Status, Streaming}; + +use crate::ServerState; +use crate::auth::principal::Principal; + +/// gRPC path prefix of the supervisor host service. +pub const SUPERVISOR_HOST_PATH_PREFIX: &str = "/openshell.supervisor.host.v1.SupervisorHost/"; + +#[derive(Clone)] +pub struct SupervisorHostService { + state: Arc, +} + +impl SupervisorHostService { + #[must_use] + pub fn new(state: Arc) -> Self { + Self { state } + } +} + +#[tonic::async_trait] +impl SupervisorHost for SupervisorHostService { + type AttachStream = Pin< + Box> + Send + 'static>, + >; + + async fn attach( + &self, + request: Request>, + ) -> Result, Status> { + let identity = match request.extensions().get::() { + Some(Principal::Supervisor(supervisor)) => supervisor.identity.clone(), + _ => { + return Err(Status::permission_denied( + "the supervisor assignment stream requires a supervisor token", + )); + } + }; + let controller = self + .state + .compute + .supervisor_host() + .ok_or_else(|| { + Status::failed_precondition("this gateway does not host a shared supervisor") + })? + .clone(); + let outbound = controller.attach(identity, request.into_inner()).await?; + Ok(Response::new(Box::pin(outbound))) + } +} diff --git a/crates/openshell-server/src/grpc/workspace.rs b/crates/openshell-server/src/grpc/workspace.rs index d2063e988c..56156f7d3f 100644 --- a/crates/openshell-server/src/grpc/workspace.rs +++ b/crates/openshell-server/src/grpc/workspace.rs @@ -68,8 +68,8 @@ fn membership_filter_subject<'a>( } } Principal::Sandbox(_) | Principal::Operator(_) => Ok(None), - Principal::Peer(_) => Err(Status::permission_denied( - "gateway peer principals cannot list workspaces", + Principal::Peer(_) | Principal::Supervisor(_) => Err(Status::permission_denied( + "gateway peer and supervisor principals cannot list workspaces", )), Principal::Anonymous => Err(Status::unauthenticated("authentication required")), } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index c0a83f4e33..f1f991e4ae 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -73,7 +73,7 @@ use std::path::{Path, PathBuf}; #[cfg(test)] use std::sync::LazyLock; use std::sync::{ - Arc, Mutex, + Arc, Mutex, OnceLock, atomic::{AtomicBool, Ordering}, }; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -241,6 +241,53 @@ fn spawn_gateway_extension_token_refresh( } }); } + +/// Mint the shared supervisor's token and renew it at half its lifetime so +/// the delivered copy is never close to expiry when the supervisor restarts +/// or reconnects. Minting failures are logged and retried shortly. +fn spawn_supervisor_token_refresh( + authority: Arc, + ttl: Duration, + controller: openshell_supervisor_control::SupervisorHostController, + mut shutdown_rx: watch::Receiver, +) { + tokio::spawn(async move { + loop { + let delay = match authority.mint_supervisor_token(ttl) { + Ok(minted) => { + let remaining = minted.expires_at.saturating_sub( + i64::try_from( + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_secs()), + ) + .unwrap_or(i64::MAX), + ); + controller.publish_credential( + openshell_supervisor_control::SupervisorCredential { + token: minted.token, + expires_at: minted.expires_at, + }, + ); + Duration::from_secs(u64::try_from(remaining).unwrap_or(0) / 2) + .max(Duration::from_secs(1)) + } + Err(error) => { + warn!(%error, "failed to mint the supervisor token; retrying"); + Duration::from_secs(10) + } + }; + tokio::select! { + () = tokio::time::sleep(delay) => {} + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + return; + } + } + } + } + }); +} pub use auth::audit::ConnectionSource; pub use multiplex::{MultiplexService, MultiplexedService}; pub use persistence::Store; @@ -744,6 +791,37 @@ pub(crate) async fn run_server( if let Some(issuer) = extension_jwt_issuer { spawn_gateway_extension_token_refresh(issuer, gateway_extension_credentials); } + // The shared supervisor authenticates with a token this gateway mints + // from its launch-signing key and carries only over TLS, so the topology + // needs both. + if let Some(controller) = state.compute.supervisor_host() { + let (authority, jwt) = state + .sandbox_session_jwt_authority + .clone() + .zip(config.gateway_jwt.as_ref()) + .ok_or_else(|| { + Error::config( + "the shared supervisor requires gateway_jwt signing material to mint \ + supervisor tokens; configure [openshell.gateway.gateway_jwt] or disable \ + supervisor_topology", + ) + })?; + if config.tls.is_none() { + return Err(Error::config( + "the shared supervisor requires gateway TLS; its token travels only over TLS. \ + Enable TLS or disable supervisor_topology", + )); + } + let ttl = jwt.token_ttl().clamp( + openshell_core::jwt::MIN_SESSION_TOKEN_TTL, + openshell_core::jwt::MAX_SESSION_TOKEN_TTL, + ); + info!( + ttl_secs = ttl.as_secs(), + "shared supervisor assignment stream enabled; minting supervisor tokens" + ); + spawn_supervisor_token_refresh(authority, ttl, controller.clone(), shutdown_rx.clone()); + } if state.sandbox_session_jwt_authority.is_some() && state.compute.supports_sandbox_authentication() @@ -1617,9 +1695,22 @@ impl ComputeDriverConfigContext<'_> { pub struct ComputeDriverBuildContext<'a> { config: ComputeDriverConfigContext<'a>, shutdown_rx: watch::Receiver, + shared_supervision: &'a OnceLock, } impl ComputeDriverBuildContext<'_> { + /// The gateway's coordination of a shared supervisor host, for an + /// in-process driver whose configuration places new generations on it. + /// The gateway owns the desired set, serves it to the host over the + /// private assignment stream, mints the supervisor token, and publishes + /// each renewal for the driver to deliver. Created on first request; every + /// call returns the same coordination. + pub fn shared_supervision(&self) -> openshell_supervisor_control::SharedSupervision { + self.shared_supervision + .get_or_init(openshell_supervisor_control::SharedSupervision::new) + .clone() + } + #[must_use] pub fn config_context(&self) -> ComputeDriverConfigContext<'_> { self.config @@ -1737,6 +1828,7 @@ async fn build_compute_runtime( info!(driver = %driver.name(), "Using compute driver"); let runtime = match driver { ConfiguredComputeDriver::Registered(registration) => { + let shared_supervision = OnceLock::new(); let build_context = ComputeDriverBuildContext { config: ComputeDriverConfigContext { driver_name: ®istration.name, @@ -1746,8 +1838,12 @@ async fn build_compute_runtime( driver_startup, }, shutdown_rx, + shared_supervision: &shared_supervision, }; let instance = registration.factory.build(build_context).await?; + let supervisor_host = shared_supervision + .into_inner() + .map(|supervision| supervision.controller().clone()); match instance { ComputeDriverInstance::InProcess(driver) => ComputeRuntime::from_driver( registration.name, @@ -1762,7 +1858,13 @@ async fn build_compute_runtime( .await .map_err(|error| { Error::execution(format!("failed to create compute runtime: {error}")) - })?, + })? + .with_supervisor_host(supervisor_host), + ComputeDriverInstance::ManagedRemote(_) if supervisor_host.is_some() => { + return Err(Error::config( + "only an in-process compute driver can use the gateway's shared supervision", + )); + } ComputeDriverInstance::ManagedRemote(mut endpoint) => { endpoint.name = registration.name; ComputeRuntime::new_remote_driver( diff --git a/crates/openshell-server/src/multiplex.rs b/crates/openshell-server/src/multiplex.rs index cc08e2ebc6..baf9b53ef3 100644 --- a/crates/openshell-server/src/multiplex.rs +++ b/crates/openshell-server/src/multiplex.rs @@ -18,6 +18,7 @@ use hyper_util::{ }; use metrics::{counter, histogram}; use openshell_core::proto::open_shell_server::OpenShellServer; +use openshell_core::proto::supervisor_host::v1::supervisor_host_server::SupervisorHostServer; use openshell_core::{ Config, proto::{Provider, UpdateProviderRequest}, @@ -294,6 +295,15 @@ impl MultiplexService { self.state.gateway_interceptors.clone(), Some(self.state.clone()), ); + // The private host stream carries launch secrets, so it bypasses the + // configurable interceptors. + let openshell = SupervisorHostRoute::new( + SupervisorHostServer::new(crate::grpc::supervisor_host::SupervisorHostService::new( + self.state.clone(), + )) + .max_decoding_message_size(MAX_GRPC_DECODE_SIZE), + openshell, + ); let reflection = self.state.reflection_service.clone(); let authz_policy = self.state.config.oidc.as_ref().map(|oidc| AuthzPolicy { admin_role: oidc.admin_role.clone(), @@ -371,6 +381,57 @@ impl MultiplexService { } } +/// Routes the private supervisor host service past the gateway interceptors +/// and every other gRPC path to the `OpenShell` service. +#[derive(Clone)] +struct SupervisorHostRoute { + host: SupervisorHostServer, + gateway: N, +} + +impl SupervisorHostRoute { + fn new( + host: SupervisorHostServer, + gateway: N, + ) -> Self { + Self { host, gateway } + } +} + +impl tower::Service> for SupervisorHostRoute +where + N: tower::Service, Response = Response, Error = Infallible> + + Clone + + Send + + 'static, + N::Future: Send + 'static, +{ + type Response = Response; + type Error = Infallible; + type Future = Pin> + Send>>; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.gateway.poll_ready(cx) + } + + fn call(&mut self, req: Request) -> Self::Future { + if req + .uri() + .path() + .starts_with(crate::grpc::supervisor_host::SUPERVISOR_HOST_PATH_PREFIX) + { + let mut host = self.host.clone(); + Box::pin(async move { + as tower::Service>>::call(&mut host, req) + .await + }) + } else { + let mut gateway = self.gateway.clone(); + Box::pin(async move { gateway.ready().await?.call(req).await }) + } + } +} + /// `OpenShell` gRPC wrapper that applies configured gateway interceptors before /// tonic dispatches to a specific RPC handler. #[derive(Clone)] @@ -754,6 +815,11 @@ fn gateway_principal_fields(principal: &Principal) -> BTreeMap { fields.insert("replica_id".to_string(), peer.replica_id.clone()); fields.insert("pod_uid".to_string(), peer.pod_uid.clone()); } + Principal::Supervisor(supervisor) => { + fields.insert("kind".to_string(), "supervisor".to_string()); + fields.insert("subject".to_string(), supervisor.identity.subject.clone()); + fields.insert("token_id".to_string(), supervisor.identity.token_id.clone()); + } Principal::Anonymous => { fields.insert("kind".to_string(), "anonymous".to_string()); } @@ -980,10 +1046,12 @@ where /// 2. `ComputeDriverAuthenticator` (path-scoped to `IssueSandboxToken`) /// — delegates a driver-native credential and receives a sandbox identity /// so the handler can mint a gateway JWT. No-op on every other path. -/// 3. `SandboxSessionJwtAuthenticator` — validates generation-bound gateway +/// 3. `SupervisorTokenAuthenticator` — validates the shared supervisor's +/// gateway-minted token (its own `typ`); applies to no other bearer. +/// 4. `SandboxSessionJwtAuthenticator` — validates generation-bound gateway /// JWTs against the durable sandbox identity. Untyped legacy sandbox JWTs /// are not admitted. -/// 4. `OidcAuthenticator` — validates user Bearer tokens against the +/// 5. `OidcAuthenticator` — validates user Bearer tokens against the /// configured OIDC issuer. Returns `Unauthenticated` for missing /// Bearer headers so non-OIDC clients can't sneak through. /// @@ -1006,6 +1074,9 @@ fn build_authenticator_chain(state: &ServerState) -> Option authenticators.push(driver); } if let Some(authority) = state.sandbox_session_jwt_authority.clone() { + authenticators.push(Arc::new( + crate::auth::supervisor_token::SupervisorTokenAuthenticator::new(authority.clone()), + )); authenticators.push(Arc::new( crate::auth::sandbox_jwt::SandboxSessionJwtAuthenticator::new( authority, @@ -1270,12 +1341,22 @@ where Principal::Peer(_) if !crate::auth::method_authz::is_peer_callable(&path) => { method_denied("gateway peer principals may not call this method") } + Principal::Supervisor(_) + if !crate::auth::method_authz::is_supervisor_callable(&path) => + { + method_denied( + "supervisor tokens may call only the supervisor assignment stream", + ) + } Principal::Anonymous => { return Ok(status_response(tonic::Status::unauthenticated( "anonymous callers may not call authenticated methods", ))); } - Principal::Operator(_) | Principal::Sandbox(_) | Principal::Peer(_) => None, + Principal::Operator(_) + | Principal::Sandbox(_) + | Principal::Peer(_) + | Principal::Supervisor(_) => None, }; if let Some((denial, status)) = denial { audit::emit_authorization_denial_for(&path, &origin, &principal, denial); @@ -3481,6 +3562,111 @@ mod tests { } } + #[tokio::test] + async fn supervisor_tokens_reach_only_the_assignment_stream() { + let attach = "/openshell.supervisor.host.v1.SupervisorHost/Attach"; + let supervisor = Principal::Supervisor(crate::auth::principal::SupervisorPrincipal { + identity: openshell_supervisor_control::HostIdentity { + subject: "openshell-supervisor:test".to_string(), + token_id: "token".to_string(), + expires_at: i64::MAX, + }, + }); + let supervisor_chain = || { + AuthenticatorChain::new(vec![Arc::new(MockAuthenticator::returning(Ok( + Authentication::Accepted(supervisor.clone()), + )))]) + }; + for (method, permitted) in [ + (attach, true), + ("/openshell.v1.OpenShell/ListSandboxes", false), + ("/openshell.v1.OpenShell/ConnectSupervisor", false), + ("/openshell.v1.OpenShell/GetSandboxConfig", false), + ("/openshell.v1.OpenShell/GetProviderCredentials", false), + ("/openshell.v1.OpenShell/RefreshSandboxToken", false), + ("/openshell.v1.OpenShell/PeerRelay", false), + ("/openshell.v1.OpenShell/Unknown", false), + ] { + let (recorder, seen) = PrincipalRecorder::new(); + let response = AuthGrpcRouter::new(recorder, Some(supervisor_chain()), None) + .call(empty_request(method)) + .await + .unwrap(); + if permitted { + assert!(grpc_status(&response).is_none(), "{method}"); + assert!(matches!( + *seen.lock().unwrap(), + Some(Principal::Supervisor(_)) + )); + } else { + assert_eq!(grpc_status(&response).as_deref(), Some("7"), "{method}"); + } + } + + // No other principal reaches the stream: an mTLS user, an + // operator, a sandbox, a peer, or an unauthenticated dev user, + // even when local unauthenticated users are allowed. + let (recorder, _) = PrincipalRecorder::new(); + let response = AuthGrpcRouter::with_peer_identity( + recorder, + None, + None, + Some(mtls_identity("openshell-client")), + true, + false, + ) + .call(empty_request(attach)) + .await + .unwrap(); + assert_eq!(grpc_status(&response).as_deref(), Some("7")); + let (recorder, seen) = PrincipalRecorder::new(); + let response = AuthGrpcRouter::with_peer_identity( + recorder, + Some(AuthenticatorChain::new(vec![Arc::new( + MockAuthenticator::returning(Ok(Authentication::NotApplicable)), + )])), + None, + None, + false, + true, + ) + .call(empty_request(attach)) + .await + .unwrap(); + assert_eq!(grpc_status(&response).as_deref(), Some("7")); + assert!(seen.lock().unwrap().is_none()); + let (recorder, _) = PrincipalRecorder::new(); + let response = AuthGrpcRouter::new(recorder, None, None) + .with_operator(Some(operator_principal())) + .call(empty_request(attach)) + .await + .unwrap(); + assert_eq!(grpc_status(&response).as_deref(), Some("7")); + for principal in [ + sandbox("sandbox-a"), + Principal::Peer(crate::auth::principal::PeerPrincipal { + replica_id: "replica".into(), + pod_uid: "pod".into(), + }), + ] { + let chain = AuthenticatorChain::new(vec![Arc::new(MockAuthenticator::returning( + Ok(Authentication::Accepted(principal)), + ))]); + let (recorder, _) = PrincipalRecorder::new(); + let response = AuthGrpcRouter::new(recorder, Some(chain), None) + .call(empty_request(attach)) + .await + .unwrap(); + assert_eq!(grpc_status(&response).as_deref(), Some("7")); + } + let (recorder, _) = PrincipalRecorder::new(); + let response = AuthGrpcRouter::new(recorder, None, None) + .call(empty_request(attach)) + .await + .unwrap(); + assert_eq!(grpc_status(&response).as_deref(), Some("7")); + } + #[tokio::test] async fn user_bearer_claims_and_local_dev_cannot_export_credentials() { let mut user = mtls_identity("ordinary-user"); diff --git a/crates/openshell-server/src/storage_proto.rs b/crates/openshell-server/src/storage_proto.rs index 808c47cab3..3b902df1f9 100644 --- a/crates/openshell-server/src/storage_proto.rs +++ b/crates/openshell-server/src/storage_proto.rs @@ -535,6 +535,7 @@ mod tests { let mut compiled_method_count = 0; let mut isolation_methods = BTreeSet::new(); let mut sandbox_protocol_methods = BTreeSet::new(); + let mut supervisor_host_methods = BTreeSet::new(); for file in &public.file { let package = file.package.as_deref().unwrap_or_default(); for service in &file.service { @@ -558,6 +559,14 @@ mod tests { .expect("sandbox protocol method name") })); } + // Private gateway-to-shared-supervisor protocol: served by the + // gateway but never part of the public SDK surface. + if package == "openshell.supervisor.host.v1" { + assert_eq!(service_name, "SupervisorHost"); + supervisor_host_methods.extend(service.method.iter().map(|method| { + method.name.as_deref().expect("supervisor host method name") + })); + } if !matches!( (package, service_name), ("openshell.v1", "OpenShell") | ("openshell.inference.v1", "Inference") @@ -594,6 +603,7 @@ mod tests { sandbox_protocol_methods, BTreeSet::from(["Exchange", "Mediate"]) ); + assert_eq!(supervisor_host_methods, BTreeSet::from(["Attach"])); for signature in PROVIDER_READINESS_RPC_SIGNATURES { assert!( methods.iter().any(|method| method == signature), @@ -618,7 +628,8 @@ mod tests { + PEER_OWNER_RPC_SIGNATURES.len() + PEER_CONFIG_RPC_SIGNATURES.len() + isolation_methods.len() - + sandbox_protocol_methods.len(), + + sandbox_protocol_methods.len() + + supervisor_host_methods.len(), "classify every compiled RPC" ); assert_eq!( @@ -638,6 +649,11 @@ mod tests { + PEER_CONFIG_RPC_SIGNATURES.len() ); assert!(methods.iter().all(|method| !method.contains(".storage."))); + assert!( + methods + .iter() + .all(|method| !method.contains(".supervisor.host.")) + ); let public_closure = schema_closure(&index, public_roots); let durable_closure = diff --git a/crates/openshell-supervisor-control/Cargo.toml b/crates/openshell-supervisor-control/Cargo.toml new file mode 100644 index 0000000000..b61564c2a5 --- /dev/null +++ b/crates/openshell-supervisor-control/Cargo.toml @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "openshell-supervisor-control" +description = "Gateway-side control of a supervisor shared by many sandboxes: assignments, launch registrations, recovery, and host observations" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +openshell-core = { path = "../openshell-core", default-features = false, features = ["jwt"] } +openshell-otel = { path = "../openshell-otel" } +openshell-sandbox-backend = { path = "../openshell-sandbox-backend" } +prost-types = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } +tokio-stream = { workspace = true } +tonic = { workspace = true } +tracing = { workspace = true } + +[lints] +workspace = true diff --git a/crates/openshell-supervisor-control/README.md b/crates/openshell-supervisor-control/README.md new file mode 100644 index 0000000000..124caa5080 --- /dev/null +++ b/crates/openshell-supervisor-control/README.md @@ -0,0 +1,65 @@ +# Shared supervisor control + +`openshell-supervisor-control` is the gateway side of a supervisor shared by +many sandboxes. One shared supervisor process hosts the sandbox instances (one +running generation of a stable Sandbox each) of a gateway; the gateway owns +which instances it hosts and serves that set to it over the private +`openshell.supervisor.host.v1` stream, which the supervisor opens with a +gateway-minted supervisor token. Everything about that control relationship +that is not specific to a compute platform lives here, so a compute driver +adds a shared supervisor by adapting storage and infrastructure, not by +re-implementing the control contracts. + +## Modules + +- `controller`: `SupervisorHostController` holds the desired set of + `SandboxInstanceAssignment`s and serves it to the host. Desired state is + uninitialized until the complete set has been derived from durable records + once, so a restarted gateway never presents an empty set as authoritative; + an explicit empty set after initialization does stop instances. Full + replacements are compare-and-replace against the revision read before the + records were listed. One host session at a time: a newer connection epoch + of the same host process or a new process supersedes the current stream, a + delayed older hello is refused, and reports from superseded streams, acks + of revisions never sent, regressing acks, and oversized reports are + dropped. A stream ends when the token that opened it expires; hosted + instances keep running. At most 256 instances are assigned. +- `registration`: `SandboxInstanceRegistration` is the JSON launch material + one assignment carries and the driver persists with the generation. It + grants the supervisor exactly that generation's gateway and Sandbox Protocol + credentials. `validate` binds route, sandbox, generation, launch session, + and proxy CA together; `from_launch_material` and `from_assignment` decode + bounded material without quoting values, which include credentials. +- `status`: `SandboxInstanceStatusReport` and its phases, converted to and + from the stream's status message; an unknown phase reads as a failure. +- `coordinator`: `SharedSupervision` is what the gateway constructs and hands + to the compute driver. `launch` validates the driver's registration, + persists the encoded launch material through the driver's operation, and + only then assigns the instance; a host at capacity is refused before the + workload runs. `begin_recovery` captures + the revision before the driver lists its durable records and + `complete_recovery` reuses current assignments, reads the rest through the + driver's operation, binds each to its recorded route and sandbox, and keeps + the previous set when any read fails; a list that raced a lifecycle edit is + refused and derived again. `supervision` and `wait_for_ready` interpret the + host's observations in one place: a disconnected host is unknown, a host + that has not reconciled to the assignment yet is degraded, and only a host + that reconciled past it without hosting the instance, hosts it for another + sandbox, or reports it ended makes it unavailable. +- `credential`: `deliver_credentials` delivers the current supervisor token + and every renewal through the driver's operation, retrying a failed + delivery until it succeeds or a newer token supersedes it. +- `proto`: the private wire protocol, generated in `openshell-core`. + +## Ownership + +The gateway (`openshell-server`) constructs the `SharedSupervision` when the +selected in-process driver asks for it, keeps the controller to serve the +host's `Attach` stream and to publish the token it mints and renews at half +its lifetime, and gives the coordination to the driver. The driver +(`openshell-driver-kubernetes` today) records placements, persists and reads +back launch material, provisions the workload's path to the supervisor, checks +its own infrastructure evidence alongside the coordinator's verdict, and +delivers the token into the Secret the supervisor Pod mounts. The supervisor +(`openshell-supervisor`) consumes only the registration and status types; it +owns admission, per-instance runtime, routing, and cleanup. diff --git a/crates/openshell-supervisor-control/src/controller.rs b/crates/openshell-supervisor-control/src/controller.rs new file mode 100644 index 0000000000..2d0aa885d9 --- /dev/null +++ b/crates/openshell-supervisor-control/src/controller.rs @@ -0,0 +1,853 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Gateway side of the private supervisor host protocol +//! (`openshell.supervisor.host.v1`). +//! +//! The gateway owns the desired set of sandbox instances a shared supervisor +//! runs. The [`SharedSupervision`](crate::SharedSupervision) coordinator +//! publishes that set here on behalf of a compute driver; the gateway's gRPC +//! server hands each host stream authenticated by a supervisor token to +//! [`SupervisorHostController::attach`], which sends complete snapshots and +//! records what the host reports. The gateway also mints that token and +//! publishes each renewal here for the driver to deliver to the host. +//! +//! Desired state is uninitialized until the complete set has been derived +//! from durable records once. Nothing is sent before that, so a restarted +//! gateway can never present an empty map as authoritative and stop running +//! instances. An explicit empty snapshot, after initialization, does stop +//! them. Full resyncs are compare-and-replace against the revision read +//! before the records were listed, so a list that raced a lifecycle edit +//! never resurrects a withdrawn instance or discards a new one. +//! +//! Exactly one host serves a gateway: a new attach from a newer connection of +//! the same host process, or from a new host process, supersedes the current +//! session; reports and acks from a superseded stream are dropped, and a +//! delayed hello from an older connection of the current process is refused. + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use tokio::sync::{mpsc, watch}; +use tokio_stream::StreamExt as _; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{Status, Streaming}; + +use crate::proto::{ + AssignmentSnapshot, HostAccepted, HostGatewayHeartbeat, HostGatewayMessage, HostHello, + HostMessage, HostStatus, SandboxInstanceAssignment, SandboxInstanceStatus, + host_gateway_message, host_message, +}; + +/// Interval at which both ends send heartbeats on an idle stream. +pub const HOST_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(15); +/// Silence after which the gateway closes a host stream. +const HOST_SILENCE_TIMEOUT: Duration = Duration::from_mins(1); +/// Bound on the host sending its hello. +const HELLO_TIMEOUT: Duration = Duration::from_secs(10); +/// Bound on queueing one gateway-to-host message; a host that cannot drain +/// its stream this long is disconnected rather than stalling the session. +const SEND_TIMEOUT: Duration = Duration::from_secs(30); +/// Gateway-to-host messages queued before the stream applies backpressure. +const OUTBOUND_QUEUE: usize = 16; +/// Most sandbox instances one shared supervisor host runs. The gateway +/// refuses to assign beyond it, and the host refuses snapshots beyond it, so +/// a sandbox is never created that its host could not accept. +pub const MAX_ASSIGNED_INSTANCES: usize = 256; +/// Most instances one host status report may describe: every assignment plus +/// instances still being removed. +pub const MAX_REPORTED_INSTANCES: usize = 4 * MAX_ASSIGNED_INSTANCES; + +/// Verified identity of a connected supervisor host: the claims of the +/// supervisor token that authenticated its stream. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct HostIdentity { + /// Subject of the supervisor token (`openshell-supervisor:`). + pub subject: String, + /// `jti` of the supervisor token. + pub token_id: String, + /// `exp` of the supervisor token (Unix seconds). The gateway ends the + /// stream when it passes; the host reconnects with a renewed token. + pub expires_at: i64, +} + +/// The current supervisor token, as the gateway mints and renews it. +/// +/// A compute driver delivers it to the supervisor process by whatever means +/// its deployment packaging provides; the Kubernetes driver writes a Secret. +#[derive(Clone, Debug)] +pub struct SupervisorCredential { + pub token: openshell_core::jwt::SecretJwt, + /// Unix seconds. + pub expires_at: i64, +} + +/// What the gateway knows about one assigned route on the host. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ObservedInstance { + /// No host session is connected. + NoHost, + /// A host is connected but has not reconciled to the snapshot that + /// assigned this route. + Pending, + /// The host reconciled to that snapshot (or the route is not assigned) + /// and does not host the route. + Absent, + /// The host's latest report of the instance. + Reported(SandboxInstanceStatus), +} + +struct Assigned { + assignment: SandboxInstanceAssignment, + /// Snapshot revision that first carried this exact assignment. + revision: u64, +} + +#[derive(Default)] +struct Desired { + initialized: bool, + revision: u64, + assignments: BTreeMap, + /// Edits made before initialization, applied over the first full list. + pending: BTreeMap>, +} + +impl Desired { + fn snapshot(&self) -> Option { + self.initialized.then(|| AssignmentSnapshot { + revision: self.revision, + instances: self + .assignments + .values() + .map(|assigned| assigned.assignment.clone()) + .collect(), + }) + } + + /// Insert or replace one assignment; returns whether anything changed. + fn put(&mut self, assignment: SandboxInstanceAssignment, revision: u64) -> bool { + match self.assignments.get(&assignment.route) { + Some(existing) if existing.assignment == assignment => false, + _ => { + self.assignments.insert( + assignment.route.clone(), + Assigned { + assignment, + revision, + }, + ); + true + } + } + } +} + +struct HostSession { + id: u64, + identity: HostIdentity, + instance_id: String, + connection_epoch: u64, + superseded: watch::Sender, + /// Highest snapshot revision sent on this stream; acks beyond it are + /// impossible and rejected. + sent_revision: u64, +} + +#[derive(Default)] +struct Observed { + session: Option, + /// Latest report of the current session. + status: Option, + next_session: u64, +} + +struct Inner { + desired: Mutex, + /// Latest supervisor token the gateway minted; `None` until the first. + credential: watch::Sender>, + /// Current desired revision; 0 while uninitialized. + revision: watch::Sender, + observed: Mutex, +} + +/// Gateway-owned desired assignments and observed status of one shared +/// supervisor host. +#[derive(Clone)] +pub struct SupervisorHostController { + inner: Arc, +} + +impl Default for SupervisorHostController { + fn default() -> Self { + Self::new() + } +} + +impl std::fmt::Debug for SupervisorHostController { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("SupervisorHostController") + .finish_non_exhaustive() + } +} + +/// Why a change to the desired set was refused. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum AssignmentError { + /// A full resync was derived from a desired set that has since changed: + /// `baseline` is the revision the caller read before deriving its list and + /// `current` the one now in force; derive the list again from it. + #[error("desired assignments changed from revision {baseline} to {current} during the resync")] + Conflict { baseline: u64, current: u64 }, + /// The change would assign more instances than one host runs. + #[error("the shared supervisor hosts at most {MAX_ASSIGNED_INSTANCES} sandbox instances")] + Capacity, +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +impl SupervisorHostController { + #[must_use] + pub fn new() -> Self { + Self { + inner: Arc::new(Inner { + desired: Mutex::new(Desired::default()), + credential: watch::Sender::new(None), + revision: watch::Sender::new(0), + observed: Mutex::new(Observed::default()), + }), + } + } + + /// Publish a freshly minted supervisor token for delivery to the host. + pub fn publish_credential(&self, credential: SupervisorCredential) { + self.inner.credential.send_replace(Some(credential)); + } + + /// The current supervisor token and every renewal after it. + #[must_use] + pub fn credential(&self) -> watch::Receiver> { + self.inner.credential.subscribe() + } + + /// The complete desired set has been derived from durable records at + /// least once. + #[must_use] + pub fn is_initialized(&self) -> bool { + lock(&self.inner.desired).initialized + } + + /// Current desired revision: 0 while uninitialized. Read it before + /// deriving a full list for [`Self::replace_assignments`]; the + /// coordinator does so through + /// [`SharedSupervision::begin_recovery`](crate::SharedSupervision::begin_recovery). + #[must_use] + pub fn revision(&self) -> u64 { + lock(&self.inner.desired).revision + } + + /// The current assignment of `route`, if any. + #[must_use] + pub fn assignment(&self, route: &str) -> Option { + lock(&self.inner.desired) + .assignments + .get(route) + .map(|assigned| assigned.assignment.clone()) + } + + /// Replace the whole desired set with `assignments`, derived from + /// durable records while the revision was `baseline`. + /// + /// Initializes the controller; edits made before initialization are + /// applied on top of the list. After initialization the replacement is + /// compare-and-replace: when the revision moved past `baseline` while + /// the list was being derived, nothing changes and the conflict is + /// returned so the caller derives the list again. + pub fn replace_assignments( + &self, + baseline: u64, + assignments: impl IntoIterator, + ) -> Result { + let mut desired = lock(&self.inner.desired); + if desired.initialized && desired.revision != baseline { + return Err(AssignmentError::Conflict { + baseline, + current: desired.revision, + }); + } + // Resolve the list and the pending overlay first, so a refused + // replacement leaves the current set untouched. + let mut resolved = BTreeMap::new(); + for assignment in assignments { + resolved.insert(assignment.route.clone(), assignment); + } + for (route, edit) in &desired.pending { + match edit { + Some(assignment) => { + resolved.insert(route.clone(), assignment.clone()); + } + None => { + resolved.remove(route); + } + } + } + if resolved.len() > MAX_ASSIGNED_INSTANCES { + return Err(AssignmentError::Capacity); + } + desired.pending.clear(); + let next = desired.revision + 1; + let mut changed = !desired.initialized; + for assignment in resolved.values() { + changed |= desired.put(assignment.clone(), next); + } + let before = desired.assignments.len(); + desired + .assignments + .retain(|route, _| resolved.contains_key(route)); + changed |= desired.assignments.len() != before; + desired.initialized = true; + if changed { + desired.revision = next; + self.inner.revision.send_replace(next); + } + Ok(desired.revision) + } + + /// Assign one instance. Before initialization the edit is kept until the + /// first full list arrives. Refused when it would exceed the host's + /// capacity, so the caller fails its create before anything runs. + pub fn assign(&self, assignment: SandboxInstanceAssignment) -> Result<(), AssignmentError> { + let mut desired = lock(&self.inner.desired); + if !desired.initialized { + let pending_additions = desired + .pending + .values() + .filter(|edit| edit.is_some()) + .count(); + if !desired.pending.contains_key(&assignment.route) + && pending_additions >= MAX_ASSIGNED_INSTANCES + { + return Err(AssignmentError::Capacity); + } + desired + .pending + .insert(assignment.route.clone(), Some(assignment)); + return Ok(()); + } + if !desired.assignments.contains_key(&assignment.route) + && desired.assignments.len() >= MAX_ASSIGNED_INSTANCES + { + return Err(AssignmentError::Capacity); + } + let next = desired.revision + 1; + if desired.put(assignment, next) { + desired.revision = next; + self.inner.revision.send_replace(next); + } + Ok(()) + } + + /// Withdraw one instance so the host stops it. + pub fn unassign(&self, route: &str) { + let mut desired = lock(&self.inner.desired); + if !desired.initialized { + desired.pending.insert(route.to_string(), None); + return; + } + if desired.assignments.remove(route).is_some() { + desired.revision += 1; + let revision = desired.revision; + self.inner.revision.send_replace(revision); + } + } + + /// Whether a host session is currently connected. + #[must_use] + pub fn host_connected(&self) -> bool { + lock(&self.inner.observed).session.is_some() + } + + /// What the connected host reports for `route`. + #[must_use] + pub fn observed(&self, route: &str) -> ObservedInstance { + let desired = lock(&self.inner.desired); + let observed = lock(&self.inner.observed); + if observed.session.is_none() { + return ObservedInstance::NoHost; + } + let Some(status) = &observed.status else { + return ObservedInstance::Pending; + }; + if let Some(instance) = status + .instances + .iter() + .find(|instance| instance.route == route) + { + return ObservedInstance::Reported(instance.clone()); + } + match desired.assignments.get(route) { + Some(assigned) if assigned.revision > status.applied_revision => { + ObservedInstance::Pending + } + _ => ObservedInstance::Absent, + } + } + + /// Serve one authenticated host stream. Reads the hello, supersedes any + /// current session, and returns the gateway-to-host stream. + pub async fn attach( + &self, + identity: HostIdentity, + mut inbound: Streaming, + ) -> Result>, Status> { + let hello = match tokio::time::timeout(HELLO_TIMEOUT, inbound.next()).await { + Ok(Some(Ok(HostMessage { + payload: Some(host_message::Payload::Hello(hello)), + }))) => hello, + Ok(Some(Ok(_))) => { + return Err(Status::invalid_argument( + "supervisor host stream must start with a hello", + )); + } + Ok(Some(Err(status))) => return Err(status), + Ok(None) => { + return Err(Status::invalid_argument( + "supervisor host stream ended before its hello", + )); + } + Err(_) => { + return Err(Status::deadline_exceeded( + "supervisor host did not send its hello in time", + )); + } + }; + if hello.instance_id.is_empty() { + return Err(Status::invalid_argument( + "supervisor host hello names no instance", + )); + } + let (tx, rx) = mpsc::channel(OUTBOUND_QUEUE); + let expires_at = identity.expires_at; + let (session_id, superseded) = self.begin_session(identity, &hello)?; + tx.send(Ok(HostGatewayMessage { + payload: Some(host_gateway_message::Payload::Accepted(HostAccepted { + session_id: session_id.to_string(), + heartbeat_interval: Some(prost_types::Duration { + seconds: i64::try_from(HOST_HEARTBEAT_INTERVAL.as_secs()).unwrap_or(15), + nanos: 0, + }), + })), + })) + .await + .map_err(|_| Status::unavailable("supervisor host stream closed"))?; + let controller = self.clone(); + tokio::spawn(async move { + controller + .run_session(session_id, expires_at, inbound, tx, superseded) + .await; + controller.end_session(session_id); + }); + Ok(ReceiverStream::new(rx)) + } + + /// Start a session for `hello`, superseding the current one. A hello from + /// the current host process with a connection epoch that is not newer is + /// a delayed or replayed connection and is refused. + fn begin_session( + &self, + identity: HostIdentity, + hello: &HostHello, + ) -> Result<(u64, watch::Receiver), Status> { + let mut observed = lock(&self.inner.observed); + if let Some(current) = &observed.session + && current.instance_id == hello.instance_id + && hello.connection_epoch <= current.connection_epoch + { + return Err(Status::failed_precondition( + "supervisor host connection epoch is not newer than the current session", + )); + } + observed.next_session += 1; + let id = observed.next_session; + if let Some(previous) = observed.session.take() { + tracing::info!( + subject = %previous.identity.subject, + instance_id = %previous.instance_id, + connection_epoch = previous.connection_epoch, + "superseding supervisor host session" + ); + previous.superseded.send_replace(true); + } + tracing::info!( + subject = %identity.subject, + instance_id = %hello.instance_id, + connection_epoch = hello.connection_epoch, + session = id, + "supervisor host attached" + ); + let (superseded, watch) = watch::channel(false); + observed.session = Some(HostSession { + id, + identity, + instance_id: hello.instance_id.clone(), + connection_epoch: hello.connection_epoch, + superseded, + sent_revision: 0, + }); + observed.status = None; + Ok((id, watch)) + } + + fn end_session(&self, id: u64) { + let mut observed = lock(&self.inner.observed); + if observed + .session + .as_ref() + .is_some_and(|session| session.id == id) + { + let session = observed.session.take(); + observed.status = None; + if let Some(session) = session { + tracing::info!( + subject = %session.identity.subject, + instance_id = %session.instance_id, + session = id, + "supervisor host detached; hosted instances keep running" + ); + } + } + } + + /// Note that `revision` was sent on session `id`. + fn record_sent(&self, id: u64, revision: u64) { + let mut observed = lock(&self.inner.observed); + if let Some(session) = observed.session.as_mut().filter(|session| session.id == id) { + session.sent_revision = session.sent_revision.max(revision); + } + } + + /// Record a report from session `id`. Reports from other sessions, acks + /// of revisions never sent on this session, regressing acks, and + /// oversized reports are dropped. + fn record_status(&self, id: u64, status: HostStatus) -> Result<(), &'static str> { + let mut observed = lock(&self.inner.observed); + let Some(session) = observed.session.as_ref().filter(|session| session.id == id) else { + return Err("report from a superseded supervisor host session"); + }; + if status.applied_revision > session.sent_revision { + return Err("supervisor host acknowledged a revision it was never sent"); + } + if status.instances.len() > MAX_REPORTED_INSTANCES { + return Err("supervisor host status reports too many instances"); + } + if observed + .status + .as_ref() + .is_some_and(|previous| status.applied_revision < previous.applied_revision) + { + return Err("supervisor host acknowledged an older revision than before"); + } + observed.status = Some(status); + Ok(()) + } + + fn snapshot(&self) -> Option { + lock(&self.inner.desired).snapshot() + } + + async fn run_session( + &self, + session_id: u64, + expires_at: i64, + mut inbound: Streaming, + tx: mpsc::Sender>, + mut superseded: watch::Receiver, + ) { + // A bearer is verified once per stream, so the stream lives no longer + // than the token did. Hosted instances are unaffected; the host + // reconnects with the renewed token it has been delivered. + let credential_expiry = tokio::time::sleep(credential_remaining(expires_at)); + tokio::pin!(credential_expiry); + let mut revision = self.inner.revision.subscribe(); + let mut sent_revision = 0; + let mut last_heard = tokio::time::Instant::now(); + let mut heartbeat = tokio::time::interval(HOST_HEARTBEAT_INTERVAL); + heartbeat.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + // A send that cannot complete within the bound ends the session, so a + // stalled host can neither block supersession nor delay heartbeats. + let send = |message: host_gateway_message::Payload| { + let tx = tx.clone(); + async move { + tx.send_timeout( + Ok(HostGatewayMessage { + payload: Some(message), + }), + SEND_TIMEOUT, + ) + .await + .is_ok() + } + }; + loop { + let current = *revision.borrow_and_update(); + if current > sent_revision { + let Some(snapshot) = self.snapshot() else { + continue; + }; + tracing::debug!( + session = session_id, + revision = snapshot.revision, + instances = snapshot.instances.len(), + "sending supervisor host assignment snapshot" + ); + let revision_sent = snapshot.revision; + if !send(host_gateway_message::Payload::Assignments(snapshot)).await { + return; + } + self.record_sent(session_id, revision_sent); + sent_revision = current; + } + tokio::select! { + changed = revision.changed() => { + if changed.is_err() { + return; + } + } + message = inbound.next() => match message { + Some(Ok(HostMessage { payload: Some(host_message::Payload::Status(status)) })) => { + last_heard = tokio::time::Instant::now(); + if let Err(reason) = self.record_status(session_id, status) { + tracing::warn!(session = session_id, reason, "dropped supervisor host status report"); + } + } + Some(Ok(HostMessage { payload: Some(host_message::Payload::Heartbeat(_)) })) => { + last_heard = tokio::time::Instant::now(); + } + Some(Ok(_)) => { + let _ = tx + .send_timeout( + Err(Status::invalid_argument( + "unexpected message on supervisor host stream", + )), + SEND_TIMEOUT, + ) + .await; + return; + } + Some(Err(status)) => { + tracing::debug!(session = session_id, %status, "supervisor host stream failed"); + return; + } + None => return, + }, + _ = heartbeat.tick() => { + if last_heard.elapsed() > HOST_SILENCE_TIMEOUT { + tracing::warn!(session = session_id, "supervisor host stream went silent; closing it"); + return; + } + if !send(host_gateway_message::Payload::Heartbeat(HostGatewayHeartbeat {})).await { + return; + } + } + _ = superseded.changed() => return, + () = &mut credential_expiry => { + tracing::info!(session = session_id, "supervisor token expired; closing the stream for reconnection"); + return; + } + } + } + } +} + +/// Time until `expires_at` (Unix seconds); zero once it has passed. +fn credential_remaining(expires_at: i64) -> Duration { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| { + i64::try_from(elapsed.as_secs()).unwrap_or(i64::MAX) + }); + Duration::from_secs(u64::try_from(expires_at.saturating_sub(now)).unwrap_or(0)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn assignment(route: &str, material: &[u8]) -> SandboxInstanceAssignment { + SandboxInstanceAssignment { + route: route.to_string(), + sandbox_id: format!("sandbox-{route}"), + generation: "gen".to_string(), + launch_material: material.to_vec(), + } + } + + fn status(applied_revision: u64, routes: &[&str]) -> HostStatus { + HostStatus { + applied_revision, + instances: routes + .iter() + .map(|route| SandboxInstanceStatus { + route: (*route).to_string(), + sandbox_id: format!("sandbox-{route}"), + phase: 2, + ready: true, + exit_code: 0, + error: String::new(), + }) + .collect(), + } + } + + #[test] + fn full_resyncs_initialize_once_and_then_compare_and_replace() { + let controller = SupervisorHostController::new(); + let revision = controller.inner.revision.subscribe(); + // Edits before the first full list wait for it. + controller + .assign(assignment("rt-created-before-list", b"a")) + .unwrap(); + controller.unassign("rt-stopped-before-list"); + assert!(!controller.is_initialized()); + assert!(controller.snapshot().is_none()); + assert!(!revision.has_changed().unwrap()); + + let baseline = controller.revision(); + assert_eq!( + controller.replace_assignments( + baseline, + [ + assignment("rt-stopped-before-list", b"old"), + assignment("rt-listed", b"b"), + ], + ), + Ok(1) + ); + let snapshot = controller.snapshot().unwrap(); + assert_eq!(snapshot.revision, 1); + assert_eq!( + snapshot + .instances + .iter() + .map(|instance| instance.route.as_str()) + .collect::>(), + ["rt-created-before-list", "rt-listed"] + ); + assert!(revision.has_changed().unwrap()); + + // An identical list changes nothing. + assert_eq!( + controller.replace_assignments( + 1, + [ + assignment("rt-created-before-list", b"a"), + assignment("rt-listed", b"b"), + ], + ), + Ok(1) + ); + controller.assign(assignment("rt-listed", b"b")).unwrap(); + assert_eq!(controller.revision(), 1); + + // A list derived at revision 1 that races a stop cannot resurrect the + // stopped instance; the caller derives it again. + controller.unassign("rt-listed"); + assert_eq!(controller.revision(), 2); + assert_eq!( + controller.replace_assignments( + 1, + [ + assignment("rt-created-before-list", b"a"), + assignment("rt-listed", b"b"), + ], + ), + Err(AssignmentError::Conflict { + baseline: 1, + current: 2 + }) + ); + assert!(controller.assignment("rt-listed").is_none()); + assert_eq!(controller.replace_assignments(2, []), Ok(3)); + assert!(controller.snapshot().unwrap().instances.is_empty()); + + // The gateway refuses the assignment its host could not accept. + let full = + (0..MAX_ASSIGNED_INSTANCES).map(|index| assignment(&format!("rt-{index}"), b"x")); + assert_eq!(controller.replace_assignments(3, full), Ok(4)); + assert_eq!( + controller.assign(assignment("rt-one-too-many", b"x")), + Err(AssignmentError::Capacity) + ); + assert_eq!(controller.assign(assignment("rt-0", b"x")), Ok(())); + } + + #[test] + fn sessions_fence_stale_streams_epochs_and_acks() { + let controller = SupervisorHostController::new(); + controller + .replace_assignments(0, [assignment("rt-a", b"a")]) + .unwrap(); + assert_eq!(controller.observed("rt-a"), ObservedInstance::NoHost); + + let identity = HostIdentity { + subject: "openshell-supervisor:test".to_string(), + token_id: "token".to_string(), + expires_at: i64::MAX, + }; + let hello = |instance_id: &str, connection_epoch| HostHello { + instance_id: instance_id.to_string(), + connection_epoch, + }; + let (first, superseded) = controller + .begin_session(identity.clone(), &hello("boot-1", 1)) + .unwrap(); + assert_eq!(controller.observed("rt-a"), ObservedInstance::Pending); + + // A delayed hello from an older connection of the same process is + // refused; a newer one supersedes the stream. + assert!( + controller + .begin_session(identity.clone(), &hello("boot-1", 1)) + .is_err() + ); + let (current, _) = controller + .begin_session(identity.clone(), &hello("boot-1", 2)) + .unwrap(); + assert!(*superseded.borrow()); + assert!(controller.record_status(first, status(0, &[])).is_err()); + + // Acks are bounded by what this stream was sent and never regress. + assert!(controller.record_status(current, status(1, &[])).is_err()); + controller.record_sent(current, 1); + controller + .record_status(current, status(1, &["rt-a"])) + .unwrap(); + assert!(controller.record_status(current, status(0, &[])).is_err()); + assert!(matches!( + controller.observed("rt-a"), + ObservedInstance::Reported(status) if status.ready + )); + + // A newer assignment is pending until the host reconciles past it. + controller.assign(assignment("rt-b", b"b")).unwrap(); + assert_eq!(controller.observed("rt-b"), ObservedInstance::Pending); + assert_eq!(controller.observed("rt-unknown"), ObservedInstance::Absent); + controller.record_sent(current, 2); + controller.record_status(current, status(2, &[])).unwrap(); + assert_eq!(controller.observed("rt-b"), ObservedInstance::Absent); + + // A new host process always takes over; ending a stale session is a no-op. + let (replacement, _) = controller + .begin_session(identity, &hello("boot-2", 1)) + .unwrap(); + controller.end_session(current); + assert!(controller.host_connected()); + controller.end_session(replacement); + assert_eq!(controller.observed("rt-a"), ObservedInstance::NoHost); + } +} diff --git a/crates/openshell-supervisor-control/src/coordinator.rs b/crates/openshell-supervisor-control/src/coordinator.rs new file mode 100644 index 0000000000..cedbb42bc2 --- /dev/null +++ b/crates/openshell-supervisor-control/src/coordinator.rs @@ -0,0 +1,390 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Gateway-owned coordination of one shared supervisor host. +//! +//! The gateway constructs a [`SharedSupervision`] and hands it to the compute +//! driver that provisions the host's sandbox instances. The driver supplies +//! each instance's registration and operations over its own durable records; +//! this module owns the contracts those operations are sequenced by: launch +//! material is persisted before an instance is assigned, a recovery baseline +//! is captured before the driver lists its records, an incomplete recovery +//! never replaces the published set, and the host's observations are +//! interpreted in one place. The driver keeps everything that is specific to +//! its infrastructure: where the records live, how its workloads reach the +//! host, and which of its own objects must exist for an instance to count as +//! supervised. + +use std::fmt::Display; +use std::future::Future; +use std::time::Duration; + +use tokio::sync::watch; + +use crate::controller::{ + AssignmentError, ObservedInstance, SupervisorCredential, SupervisorHostController, +}; +use crate::registration::{RegistrationError, SandboxInstanceRegistration}; +use crate::status::SandboxInstanceStatusReport; + +/// Interval at which [`SharedSupervision::wait_for_ready`] re-reads the +/// host's observations. +const READINESS_POLL: Duration = Duration::from_secs(1); + +/// Why a launch was not assigned. +#[derive(Debug, thiserror::Error)] +pub enum LaunchError { + /// The registration is not valid; nothing was persisted or assigned. + #[error("{0}")] + Invalid(RegistrationError), + /// The driver could not persist the launch material; nothing was assigned. + #[error("{0}")] + Persist(E), + /// The host is at capacity. The material was persisted; the caller rolls + /// the generation back before its workload runs. + #[error("{0}")] + Refused(AssignmentError), +} + +/// Revision of the desired set read before the driver listed its durable +/// records. Completing a recovery with it makes the replacement +/// compare-and-replace against that revision. +#[must_use = "complete the recovery with the records listed after this baseline was read"] +pub struct RecoveryBaseline { + revision: u64, +} + +/// A generation that may run on the shared supervisor, as the driver's +/// durable records describe it, with the record the driver needs to read its +/// persisted launch material. +pub struct RecoverableInstance { + pub route: String, + pub sandbox_id: String, + pub record: R, +} + +/// Result of a recovery pass. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RecoveryOutcome { + /// The desired set matches the records; carries the revision in force. + Published(u64), + /// The desired set changed while the records were listed; the next pass + /// derives them again. + Conflict, + /// Launch material could not be read or does not match its record, or the + /// set exceeds the host's capacity; the previous set stays published. + Kept, +} + +/// What the host's observations say about one instance's supervision. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InstanceSupervision { + /// The host supervises the instance and the gateway accepts it. + Available, + /// The host has not reconciled to the assigning snapshot yet, or hosts + /// the instance but it is still starting. + Degraded, + /// The host reconciled past the assignment and does not host the + /// instance, hosts it for another sandbox, or reports it ended. + Unavailable, + /// No host is connected, so nothing is known. + Unknown, +} + +/// The gateway's coordination of one shared supervisor host: the desired set +/// it publishes, the token it delivers, and the observations it reads back. +/// +/// Clones share one controller. The gateway keeps one for serving the host's +/// stream and minting its token, and gives one to the compute driver. +#[derive(Clone, Debug, Default)] +pub struct SharedSupervision { + controller: SupervisorHostController, +} + +impl SharedSupervision { + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// The controller the gateway serves to the host and publishes the + /// supervisor token into. + #[must_use] + pub fn controller(&self) -> &SupervisorHostController { + &self.controller + } + + /// The current supervisor token and every renewal after it, for the + /// driver to deliver to the host (see + /// [`deliver_credentials`](crate::credential::deliver_credentials)). + #[must_use] + pub fn credentials(&self) -> watch::Receiver> { + self.controller.credential() + } + + /// Register a generation's sandbox instance: validate `registration`, + /// persist its encoded launch material through `persist`, and only then + /// assign the instance so the host starts it. The launch material is + /// persisted first so a restarted gateway recovers the assignment from the + /// generation's own records. An identical assignment changes nothing. + /// Capacity is refused as [`LaunchError::Refused`], so the caller fails + /// its create before the workload runs. + pub async fn launch( + &self, + registration: &SandboxInstanceRegistration, + persist: impl FnOnce(Vec) -> F + Send, + ) -> Result<(), LaunchError> + where + E: Send, + F: Future> + Send, + { + registration.validate().map_err(LaunchError::Invalid)?; + let assignment = registration.to_assignment().map_err(LaunchError::Invalid)?; + persist(assignment.launch_material.clone()) + .await + .map_err(LaunchError::Persist)?; + self.controller + .assign(assignment) + .map_err(LaunchError::Refused) + } + + /// Withdraw a generation's sandbox instance so the host stops it. Nothing + /// happens for a route that is not assigned. + pub fn withdraw(&self, route: &str) { + self.controller.unassign(route); + } + + /// Start a recovery pass. Call it before listing the durable records the + /// pass derives the desired set from, so a lifecycle edit that lands + /// during the list is detected by [`Self::complete_recovery`] instead of + /// being undone. + pub fn begin_recovery(&self) -> RecoveryBaseline { + RecoveryBaseline { + revision: self.controller.revision(), + } + } + + /// Replace the desired set with `candidates`, the generations that may + /// run according to the records listed after `baseline` was read. Each + /// candidate's launch material comes from its current assignment or, for + /// a generation this gateway has not assigned since it started, from + /// `read` on the driver's record, which returns the persisted bytes or + /// `None` when they are gone. Material is decoded without quoting values, + /// bound to the recorded route and sandbox, and validated. + /// + /// The first completed pass initializes the desired set; until then + /// nothing is sent to the host. A failed or mismatched read keeps the + /// previous set, because an incomplete set would stop running instances. + /// A pass that raced a lifecycle edit is refused and derived again next + /// time. + pub async fn complete_recovery( + &self, + baseline: RecoveryBaseline, + candidates: Vec>, + read: impl Fn(&R) -> F + Send + Sync, + ) -> RecoveryOutcome + where + R: Send + Sync, + E: Display, + F: Future>, E>> + Send, + { + let mut assignments = Vec::with_capacity(candidates.len()); + for candidate in &candidates { + if let Some(assignment) = self.controller.assignment(&candidate.route) { + assignments.push(assignment); + continue; + } + let material = match read(&candidate.record).await { + Ok(Some(material)) => material, + Ok(None) => { + tracing::warn!( + sandbox_id = %candidate.sandbox_id, + route = %candidate.route, + "shared sandbox instance has no launch material; it stays unassigned" + ); + continue; + } + Err(error) => { + tracing::warn!( + sandbox_id = %candidate.sandbox_id, + route = %candidate.route, + %error, + "could not read shared sandbox instance launch material; keeping the previous assignments" + ); + return RecoveryOutcome::Kept; + } + }; + match recovered_assignment(candidate, &material) { + Ok(assignment) => assignments.push(assignment), + Err(error) => { + tracing::warn!( + sandbox_id = %candidate.sandbox_id, + route = %candidate.route, + %error, + "shared sandbox instance launch material is unusable; keeping the previous assignments" + ); + return RecoveryOutcome::Kept; + } + } + } + match self + .controller + .replace_assignments(baseline.revision, assignments) + { + Ok(revision) => RecoveryOutcome::Published(revision), + Err(AssignmentError::Conflict { .. }) => RecoveryOutcome::Conflict, + Err(error) => { + tracing::warn!(%error, "could not publish the shared sandbox instances; keeping the previous assignments"); + RecoveryOutcome::Kept + } + } + } + + /// What the host's observations say about the instance of `sandbox_id` + /// on `route`. Only a host that has reconciled to the assigning snapshot + /// renders a verdict; a disconnected or still-reconciling host is not + /// evidence that a running instance is gone. + #[must_use] + pub fn supervision(&self, route: &str, sandbox_id: &str) -> InstanceSupervision { + match self.controller.observed(route) { + ObservedInstance::Reported(status) => { + let status = SandboxInstanceStatusReport::from_proto(&status); + if status.sandbox_id != sandbox_id || status.phase.is_terminal() { + InstanceSupervision::Unavailable + } else if status.ready { + InstanceSupervision::Available + } else { + InstanceSupervision::Degraded + } + } + ObservedInstance::Pending => InstanceSupervision::Degraded, + ObservedInstance::Absent => InstanceSupervision::Unavailable, + ObservedInstance::NoHost => InstanceSupervision::Unknown, + } + } + + /// Wait for the instance to become available: `true` once the host + /// reports it ready, `false` once the host has reconciled past the + /// assignment without hosting it or reports it ended. A disconnected or + /// still-reconciling host is not a verdict, so the caller bounds the wait + /// with its own bootstrap grace. + pub async fn wait_for_ready(&self, route: &str, sandbox_id: &str) -> bool { + loop { + match self.supervision(route, sandbox_id) { + InstanceSupervision::Available => return true, + InstanceSupervision::Unavailable => return false, + InstanceSupervision::Degraded | InstanceSupervision::Unknown => { + tokio::time::sleep(READINESS_POLL).await; + } + } + } + } +} + +/// Decode persisted launch material for `candidate`, binding it to the +/// recorded route and sandbox before trusting it. +fn recovered_assignment( + candidate: &RecoverableInstance, + material: &[u8], +) -> Result { + let registration = SandboxInstanceRegistration::from_launch_material(material)?; + if registration.route != candidate.route || registration.sandbox_id != candidate.sandbox_id { + return Err(RegistrationError::new( + "shared sandbox instance launch material does not match the recorded generation", + )); + } + registration.validate()?; + registration.to_assignment() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::proto::SandboxInstanceAssignment; + + fn candidate(route: &str, sandbox_id: &str) -> RecoverableInstance<&'static str> { + RecoverableInstance { + route: route.to_string(), + sandbox_id: sandbox_id.to_string(), + record: "record", + } + } + + #[tokio::test] + async fn recovery_keeps_the_previous_set_unless_every_record_is_usable() { + let supervision = SharedSupervision::new(); + let assigned = SandboxInstanceAssignment { + route: "rt-assigned-0123456789".to_string(), + sandbox_id: "sandbox-a".to_string(), + generation: "gen".to_string(), + launch_material: b"opaque".to_vec(), + }; + // An assignment made before the first recovery is carried by it. + supervision.controller().assign(assigned.clone()).unwrap(); + + // A record whose material cannot be read keeps the set uninitialized. + let baseline = supervision.begin_recovery(); + let outcome = supervision + .complete_recovery( + baseline, + vec![ + candidate(&assigned.route, &assigned.sandbox_id), + candidate("rt-unreadable-0123456789", "sandbox-b"), + ], + |_record| async { Err::>, _>("boom") }, + ) + .await; + assert_eq!(outcome, RecoveryOutcome::Kept); + assert!(!supervision.controller().is_initialized()); + + // Material that is not a registration is refused without quoting it. + let baseline = supervision.begin_recovery(); + let outcome = supervision + .complete_recovery( + baseline, + vec![candidate("rt-garbage-0123456789", "sandbox-c")], + |_record| async { Ok::<_, &str>(Some(b"{\"secret\":1}".to_vec())) }, + ) + .await; + assert_eq!(outcome, RecoveryOutcome::Kept); + assert!(!supervision.controller().is_initialized()); + + // A record without material is skipped; the current assignment is + // reused without a read; the pass initializes the set. + let baseline = supervision.begin_recovery(); + let outcome = supervision + .complete_recovery( + baseline, + vec![ + candidate(&assigned.route, &assigned.sandbox_id), + candidate("rt-gone-0123456789", "sandbox-d"), + ], + |_record| async { Ok::>, &str>(None) }, + ) + .await; + assert_eq!(outcome, RecoveryOutcome::Published(1)); + assert!(supervision.controller().is_initialized()); + assert_eq!( + supervision.controller().assignment(&assigned.route), + Some(assigned.clone()) + ); + + // A baseline read before a lifecycle edit is refused, and the edit stands. + let baseline = supervision.begin_recovery(); + supervision.withdraw(&assigned.route); + let outcome = supervision + .complete_recovery( + baseline, + vec![candidate(&assigned.route, &assigned.sandbox_id)], + |_record| async { Ok::>, &str>(None) }, + ) + .await; + assert_eq!(outcome, RecoveryOutcome::Conflict); + assert!( + supervision + .controller() + .assignment(&assigned.route) + .is_none() + ); + } +} diff --git a/crates/openshell-supervisor-control/src/credential.rs b/crates/openshell-supervisor-control/src/credential.rs new file mode 100644 index 0000000000..7a2d9423e1 --- /dev/null +++ b/crates/openshell-supervisor-control/src/credential.rs @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Delivery of the gateway-minted supervisor token to the shared supervisor. +//! +//! The gateway mints and renews the token and publishes each renewal into +//! the controller. How the token reaches the supervisor process is +//! deployment packaging (the Kubernetes driver patches a Secret the +//! supervisor Pod mounts); the retry policy around that delivery is the +//! same for every packaging and lives here. + +use std::fmt::Display; +use std::future::Future; +use std::time::Duration; + +use tokio::sync::watch; + +use crate::controller::SupervisorCredential; + +/// Deliver the current supervisor token and every renewal after it. +/// +/// The initial value is delivered too. A failed delivery is retried every +/// `retry` until it succeeds or a newer token supersedes it. Returns at +/// shutdown or once the publisher is gone. +pub async fn deliver_credentials( + mut credential: watch::Receiver>, + mut shutdown: watch::Receiver, + retry: Duration, + mut deliver: F, +) where + E: Display, + F: FnMut(SupervisorCredential) -> Fut, + Fut: Future>, +{ + loop { + let current = credential.borrow_and_update().clone(); + if let Some(current) = current { + while let Err(error) = deliver(current.clone()).await { + tracing::warn!(%error, "supervisor token delivery failed; retrying"); + tokio::select! { + () = tokio::time::sleep(retry) => {} + changed = shutdown.changed() => { + if changed.is_err() || *shutdown.borrow() { + return; + } + } + } + if credential.has_changed().unwrap_or(true) { + break; + } + } + } + tokio::select! { + changed = credential.changed() => { + if changed.is_err() { + return; + } + } + changed = shutdown.changed() => { + if changed.is_err() || *shutdown.borrow() { + return; + } + } + } + } +} diff --git a/crates/openshell-supervisor-control/src/lib.rs b/crates/openshell-supervisor-control/src/lib.rs new file mode 100644 index 0000000000..4401ff3d84 --- /dev/null +++ b/crates/openshell-supervisor-control/src/lib.rs @@ -0,0 +1,52 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Gateway-side control of a supervisor shared by many sandboxes. +//! +//! One shared supervisor hosts the sandbox instances (one running generation +//! of a stable Sandbox each) of a gateway. The gateway owns which instances +//! it hosts and serves that set to it over the private +//! `openshell.supervisor.host.v1` stream, which the supervisor opens with a +//! gateway-minted supervisor token. This crate holds everything about that +//! control relationship that is not specific to a compute platform: +//! +//! - [`controller`]: the desired set, its revision fencing, the host +//! sessions, and the observations they report. +//! - [`registration`] and [`status`]: the launch material one assignment +//! carries and the status the host reports, with their validation and wire +//! mapping. +//! - [`coordinator`]: the gateway-owned [`SharedSupervision`] a compute +//! driver is handed: validated persist-before-assign launches, recovery +//! from durable records, and interpretation of observations. +//! - [`credential`]: delivery of the supervisor token with retry. +//! +//! The private wire protocol is generated in `openshell-core` and re-exported +//! as [`proto`]. The supervisor process consumes the registration and status +//! types; it never links the controller or coordinator. + +pub mod controller; +pub mod coordinator; +pub mod credential; +pub mod registration; +pub mod status; + +/// The private gateway-to-shared-supervisor protocol +/// (`openshell.supervisor.host.v1`). +pub mod proto { + pub use openshell_core::proto::supervisor_host::v1::*; +} + +pub use controller::{ + AssignmentError, HOST_HEARTBEAT_INTERVAL, HostIdentity, MAX_ASSIGNED_INSTANCES, + MAX_REPORTED_INSTANCES, ObservedInstance, SupervisorCredential, SupervisorHostController, +}; +pub use coordinator::{ + InstanceSupervision, LaunchError, RecoverableInstance, RecoveryBaseline, RecoveryOutcome, + SharedSupervision, +}; +pub use credential::deliver_credentials; +pub use registration::{ + INSTANCE_REMOVAL_GRACE, MAX_LAUNCH_MATERIAL_BYTES, ProxyCaMaterial, RegistrationError, + RegistrationTraceContext, SandboxInstanceRegistration, SecretPem, +}; +pub use status::{SandboxInstancePhaseReport, SandboxInstanceStatusReport}; diff --git a/crates/openshell-supervisor-control/src/registration.rs b/crates/openshell-supervisor-control/src/registration.rs new file mode 100644 index 0000000000..508227554f --- /dev/null +++ b/crates/openshell-supervisor-control/src/registration.rs @@ -0,0 +1,257 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Launch material of a sandbox instance hosted by a supervisor shared by +//! many sandboxes. +//! +//! A sandbox instance is one running generation of a stable Sandbox resource. +//! The gateway assigns instances to the shared supervisor over the private +//! `openshell.supervisor.host.v1` stream; each assignment carries one +//! instance's [`SandboxInstanceRegistration`] as JSON launch material, which +//! the compute driver also persists with the generation so a restarted +//! gateway can recover its assignments. A registration grants the supervisor +//! exactly the per-generation gateway and Sandbox Protocol credentials it +//! carries; it confers no other identity and is never available to a sandbox. + +use std::time::Duration; + +use openshell_sandbox_backend::boundary_protocol::{SandboxRuntimeDescriptor, SandboxTransport}; +use serde::{Deserialize, Serialize}; + +use crate::proto::SandboxInstanceAssignment; + +/// Largest launch material one assignment may carry. +pub const MAX_LAUNCH_MATERIAL_BYTES: usize = 256 * 1024; +/// Time a shared supervisor gives a withdrawn sandbox instance's workload to +/// stop before cancelling the instance. +pub const INSTANCE_REMOVAL_GRACE: Duration = Duration::from_secs(20); + +/// Why launch material was refused. The message never quotes field values, +/// which include credentials. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("{0}")] +pub struct RegistrationError(String); + +impl RegistrationError { + pub(crate) fn new(message: impl Into) -> Self { + Self(message.into()) + } +} + +/// PEM private key that never appears in `Debug` output. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(transparent)] +pub struct SecretPem(String); + +impl SecretPem { + #[must_use] + pub fn new(pem: String) -> Self { + Self(pem) + } + + #[must_use] + pub fn expose_secret(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Debug for SecretPem { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("SecretPem([REDACTED])") + } +} + +/// Proxy interception CA issued for one generation. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ProxyCaMaterial { + pub certificate_pem: String, + pub private_key_pem: SecretPem, +} + +/// W3C trace context of the operation that created the generation. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RegistrationTraceContext { + pub traceparent: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tracestate: Option, +} + +impl RegistrationTraceContext { + /// The trace context of the current span, for the generation created in + /// it; `None` outside an exported trace. + #[must_use] + pub fn current() -> Option { + let environment = openshell_otel::current_trace_context_environment(); + let value = |name: &str| { + environment + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| value.clone()) + }; + value(openshell_otel::TRACEPARENT_ENV).map(|traceparent| Self { + traceparent, + tracestate: value(openshell_otel::TRACESTATE_ENV), + }) + } +} + +/// Trusted launch material for one sandbox instance on a shared supervisor. +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SandboxInstanceRegistration { + /// Route handle the runtime names; also the instance's key on the host. + pub route: String, + pub sandbox_id: String, + pub sandbox_name: String, + /// Supervisor-side launch descriptor using the routed transport. + pub runtime_descriptor: SandboxRuntimeDescriptor, + /// Gateway-issued credentials for exactly this generation. + pub auth: openshell_core::jwt::SupervisorAuthBundle, + /// Encoded `MainProcessConfig` for the canonical process. + pub main_process_spec: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workdir: Option, + /// Workload image recorded in the instance's OCSF events. + #[serde(default)] + pub container_image: String, + pub proxy_ca: ProxyCaMaterial, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub trace_context: Option, +} + +impl std::fmt::Debug for SandboxInstanceRegistration { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("SandboxInstanceRegistration") + .field("route", &self.route) + .field("sandbox_id", &self.sandbox_id) + .field("sandbox_name", &self.sandbox_name) + .field("runtime_descriptor", &self.runtime_descriptor) + .field("auth", &self.auth) + .field("workdir", &self.workdir) + .field("container_image", &self.container_image) + .finish_non_exhaustive() + } +} + +impl SandboxInstanceRegistration { + /// Check that every part of the registration names the same sandbox, + /// generation, launch session, and route. + pub fn validate(&self) -> Result<(), RegistrationError> { + openshell_sandbox_backend::runtime_routed::validate_route_handle(&self.route) + .map_err(|error| RegistrationError::new(error.to_string()))?; + let descriptor = &self.runtime_descriptor; + match &descriptor.transport { + SandboxTransport::RuntimeInitiatedRouted { route, .. } if *route == self.route => {} + _ => { + return Err(RegistrationError::new( + "shared supervisor sandbox instances require this route's routed transport", + )); + } + } + openshell_sandbox_backend::validate_runtime_coordinates(descriptor) + .map_err(|error| RegistrationError::new(error.to_string()))?; + if self.sandbox_id.is_empty() || descriptor.boundary_id != self.sandbox_id { + return Err(RegistrationError::new( + "runtime descriptor does not match the registered sandbox", + )); + } + if descriptor.generation != self.auth.runtime_generation.as_str() + || descriptor.session_id != self.auth.session_id + { + return Err(RegistrationError::new( + "launch credentials do not match the runtime generation", + )); + } + self.auth + .validate() + .map_err(|error| RegistrationError::new(error.to_string()))?; + openshell_core::sandbox_env::MainProcessConfig::decode(&self.main_process_spec) + .map_err(RegistrationError::new)?; + if self.proxy_ca.certificate_pem.is_empty() + || self.proxy_ca.private_key_pem.expose_secret().is_empty() + { + return Err(RegistrationError::new("proxy CA material is required")); + } + Ok(()) + } + + /// Encode this registration as the launch material of one assignment. + pub fn to_assignment(&self) -> Result { + let launch_material = serde_json::to_vec(self).map_err(|error| { + RegistrationError::new(format!("encode sandbox instance registration: {error}")) + })?; + if launch_material.len() > MAX_LAUNCH_MATERIAL_BYTES { + return Err(RegistrationError::new( + "sandbox instance launch material is too large", + )); + } + Ok(SandboxInstanceAssignment { + route: self.route.clone(), + sandbox_id: self.sandbox_id.clone(), + generation: self.auth.runtime_generation.to_string(), + launch_material, + }) + } + + /// Decode persisted or transmitted launch material without validating + /// it. Decoder errors are not reported, because they can quote field + /// values, including credentials. + pub fn from_launch_material(material: &[u8]) -> Result { + if material.len() > MAX_LAUNCH_MATERIAL_BYTES { + return Err(RegistrationError::new( + "sandbox instance launch material is too large", + )); + } + serde_json::from_slice(material).map_err(|_| { + RegistrationError::new("sandbox instance launch material is not a valid registration") + }) + } + + /// Decode and validate one assignment's launch material, checking that + /// the assignment's own identifiers agree with it. + pub fn from_assignment( + assignment: &SandboxInstanceAssignment, + ) -> Result { + let registration = Self::from_launch_material(&assignment.launch_material)?; + if registration.route != assignment.route + || registration.sandbox_id != assignment.sandbox_id + || registration.auth.runtime_generation.as_str() != assignment.generation + { + return Err(RegistrationError::new( + "sandbox instance assignment does not match its launch material", + )); + } + registration.validate()?; + Ok(registration) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn assignments_must_agree_with_their_launch_material() { + let mismatched = SandboxInstanceAssignment { + route: "rt-0123456789abcdef".to_string(), + sandbox_id: "sandbox".to_string(), + generation: "gen".to_string(), + launch_material: b"{}".to_vec(), + }; + let error = SandboxInstanceRegistration::from_assignment(&mismatched).unwrap_err(); + assert!(error.to_string().contains("not a valid registration")); + let oversized = SandboxInstanceAssignment { + launch_material: vec![b' '; MAX_LAUNCH_MATERIAL_BYTES + 1], + ..mismatched + }; + assert!( + SandboxInstanceRegistration::from_assignment(&oversized) + .unwrap_err() + .to_string() + .contains("too large") + ); + } +} diff --git a/crates/openshell-supervisor-control/src/status.rs b/crates/openshell-supervisor-control/src/status.rs new file mode 100644 index 0000000000..6f5e0e20b7 --- /dev/null +++ b/crates/openshell-supervisor-control/src/status.rs @@ -0,0 +1,145 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Status of a hosted sandbox instance, as the shared supervisor reports it +//! on the private `openshell.supervisor.host.v1` stream. + +use serde::{Deserialize, Serialize}; + +use crate::proto::{SandboxInstancePhase, SandboxInstanceStatus}; + +/// Lifecycle phase of a hosted sandbox instance. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "phase", rename_all = "snake_case")] +pub enum SandboxInstancePhaseReport { + Starting, + Supervising, + Stopping, + Exited { exit_code: i32 }, + Failed { error: String }, +} + +impl SandboxInstancePhaseReport { + /// Supervision has ended and cannot resume for this generation. + #[must_use] + pub fn is_terminal(&self) -> bool { + matches!(self, Self::Exited { .. } | Self::Failed { .. }) + } +} + +/// Status of one hosted sandbox instance. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SandboxInstanceStatusReport { + pub route: String, + pub sandbox_id: String, + #[serde(flatten)] + pub phase: SandboxInstancePhaseReport, + /// The gateway currently accepts this instance's supervision. + pub ready: bool, +} + +impl SandboxInstanceStatusReport { + /// The wire form of this report. + #[must_use] + pub fn to_proto(&self) -> SandboxInstanceStatus { + let (phase, exit_code, error) = match &self.phase { + SandboxInstancePhaseReport::Starting => { + (SandboxInstancePhase::Starting, 0, String::new()) + } + SandboxInstancePhaseReport::Supervising => { + (SandboxInstancePhase::Supervising, 0, String::new()) + } + SandboxInstancePhaseReport::Stopping => { + (SandboxInstancePhase::Stopping, 0, String::new()) + } + SandboxInstancePhaseReport::Exited { exit_code } => { + (SandboxInstancePhase::Exited, *exit_code, String::new()) + } + SandboxInstancePhaseReport::Failed { error } => { + (SandboxInstancePhase::Failed, 0, error.clone()) + } + }; + SandboxInstanceStatus { + route: self.route.clone(), + sandbox_id: self.sandbox_id.clone(), + phase: phase.into(), + ready: self.ready, + exit_code, + error, + } + } + + /// Read a wire status. An unknown phase reads as a failure, so a newer + /// host can never look healthy to an older gateway. + #[must_use] + pub fn from_proto(status: &SandboxInstanceStatus) -> Self { + let phase = match SandboxInstancePhase::try_from(status.phase) { + Ok(SandboxInstancePhase::Starting) => SandboxInstancePhaseReport::Starting, + Ok(SandboxInstancePhase::Supervising) => SandboxInstancePhaseReport::Supervising, + Ok(SandboxInstancePhase::Stopping) => SandboxInstancePhaseReport::Stopping, + Ok(SandboxInstancePhase::Exited) => SandboxInstancePhaseReport::Exited { + exit_code: status.exit_code, + }, + Ok(SandboxInstancePhase::Failed) => SandboxInstancePhaseReport::Failed { + error: status.error.clone(), + }, + Ok(SandboxInstancePhase::Unspecified) | Err(_) => SandboxInstancePhaseReport::Failed { + error: "sandbox instance reported an unknown phase".to_string(), + }, + }; + let ready = status.ready && !phase.is_terminal(); + Self { + route: status.route.clone(), + sandbox_id: status.sandbox_id.clone(), + phase, + ready, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn status_reports_round_trip_and_unknown_phases_fail_closed() { + for report in [ + SandboxInstanceStatusReport { + route: "rt-a".to_string(), + sandbox_id: "sandbox-a".to_string(), + phase: SandboxInstancePhaseReport::Supervising, + ready: true, + }, + SandboxInstanceStatusReport { + route: "rt-a".to_string(), + sandbox_id: "sandbox-a".to_string(), + phase: SandboxInstancePhaseReport::Exited { exit_code: 3 }, + ready: false, + }, + SandboxInstanceStatusReport { + route: "rt-a".to_string(), + sandbox_id: "sandbox-a".to_string(), + phase: SandboxInstancePhaseReport::Failed { + error: "boom".to_string(), + }, + ready: false, + }, + ] { + assert_eq!( + SandboxInstanceStatusReport::from_proto(&report.to_proto()), + report + ); + } + let unknown = SandboxInstanceStatus { + route: "rt-a".to_string(), + sandbox_id: "sandbox-a".to_string(), + phase: 99, + ready: true, + exit_code: 0, + error: String::new(), + }; + let report = SandboxInstanceStatusReport::from_proto(&unknown); + assert!(report.phase.is_terminal()); + assert!(!report.ready); + } +} diff --git a/crates/openshell-supervisor/Cargo.toml b/crates/openshell-supervisor/Cargo.toml index a42c370404..e6e7fcd6ba 100644 --- a/crates/openshell-supervisor/Cargo.toml +++ b/crates/openshell-supervisor/Cargo.toml @@ -20,6 +20,7 @@ openshell-core = { path = "../openshell-core", default-features = false, feature openshell-extension-core = { path = "../openshell-extension-core" } openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-sandbox-backend = { path = "../openshell-sandbox-backend" } +openshell-supervisor-control = { path = "../openshell-supervisor-control" } openshell-ocsf = { path = "../openshell-ocsf" } openshell-otel = { path = "../openshell-otel" } openshell-policy = { path = "../openshell-policy" } @@ -29,6 +30,7 @@ openshell-supervisor-network = { path = "../openshell-supervisor-network", defau openshell-supervisor-process = { path = "../openshell-supervisor-process" } clap = { workspace = true } +futures = { workspace = true } miette = { workspace = true } nix = { workspace = true } opentelemetry = { workspace = true } @@ -42,6 +44,8 @@ socket2 = { workspace = true } tempfile = "3" thiserror = { workspace = true } tokio = { workspace = true } +tokio-rustls = { workspace = true } +tokio-stream = { workspace = true } tonic = { workspace = true, features = ["channel", "tls-native-roots"] } tracing = { workspace = true } tracing-appender = { workspace = true } @@ -57,14 +61,13 @@ bundled-ca-roots = ["openshell-supervisor-network/bundled-ca-roots"] [dev-dependencies] openshell-sandbox-backend = { path = "../openshell-sandbox-backend", features = ["test-support"] } -tonic-prost = { workspace = true } bytes = { workspace = true } http = { workspace = true } http-body-util = { workspace = true } hyper = { workspace = true } hyper-util = { workspace = true } +tonic-prost = { workspace = true } tokio = { workspace = true, features = ["test-util"] } -futures = { workspace = true } temp-env = "0.3" tokio-tungstenite = { workspace = true } diff --git a/crates/openshell-supervisor/README.md b/crates/openshell-supervisor/README.md index ac2f3ed888..10964b8542 100644 --- a/crates/openshell-supervisor/README.md +++ b/crates/openshell-supervisor/README.md @@ -59,14 +59,27 @@ callbacks, reconnects, and renewal therefore inherit the instance's identity, and threads outside an instance carry none and push no logs. The OCSF context also supplies the sandbox identity in middleware requests. -Removing an instance stops its workload, shuts down only its runtime, and -revokes its gateway credentials; the key stays registered until the -instance's thread has finished. An instance that panics reports a failure and +The host is the one registry of its instances. `admit` validates that the key +is a single path component, then, under the registry lock, accepts the same +launch fingerprint for an occupied key again, rejects a different one, and +rejects any admission while the key is being removed. A new instance gets an +empty private directory under the host root, and `sandbox_instance_launch` +prepares its bindings and body there. A failed admission leaves no entry and +no files. + +Removing an instance stops its workload, shuts down only its runtime, which +drops its runtime route, and revokes its gateway credentials; only then is its +private directory deleted and its key released, so a replacement can never +meet the old instance's resources. Removal runs on one detached task that +every remover joins, so a caller that disconnects cannot cancel it, and it +checks the entry's serial before releasing the key. If the directory cannot +be deleted, the key stays reserved and reports a failed phase, and the next +removal retries the deletion. An instance that panics reports a failure and is revoked the same way. An instance that is still starting is cancelled at once; the sandbox runtime then loses supervision and fails closed, as it does when a dedicated supervisor exits. A hosted instance without an explicit proxy -trust directory receives a private one for its lifetime. Hosted processes must -install `LogPushLayer::thread_bound` and an `OcsfShorthandLayer` writing to +trust directory uses its private directory. Hosted processes must install +`LogPushLayer::thread_bound` and an `OcsfShorthandLayer` writing to `SandboxInstanceDenialWriter`. ## Console logging @@ -108,15 +121,69 @@ belong to the sandbox or isolation backend. Signals use `BoundarySignal`. These portable control-plane foundations do not qualify a platform isolation backend or enable a Windows workload runtime. -## Shared host - -`openshell-supervisor --mode=shared --gateway-endpoint https://gateway:8080 --address 0.0.0.0:5501` -starts a shared-host placeholder for the Kubernetes gateway supervisor topology. -The listener at `--address` accepts and closes connections, including kubelet -`tcpSocket` probes, and reports process health only. `--gateway-endpoint` selects -the gateway the supervisor will dial for session control. The outbound connection, -control RPCs, sandbox sessions, gateway authentication, and tenant routing are not -implemented yet. Omitting `--mode` keeps the existing dedicated behavior. -`supervisor_topology = "gateway"` deploys this placeholder; sandboxes -continue using dedicated supervisors. SIGTERM and SIGINT close the listener. -Shared mode is Unix-only. +## Shared supervisor host + +`openshell-supervisor --mode=shared --gateway-endpoint --token-file --address ` +hosts the sandbox instances of many sandboxes in one process for the +Kubernetes gateway supervisor topology. It is Unix-only. The host opens no +control listener: it dials the gateway and receives its assignments from it. + +- **Gateway attachment.** The host dials `--gateway-endpoint` (which must be + `https`), verifying the gateway with `OPENSHELL_TLS_CA` and + `OPENSHELL_GATEWAY_TLS_SERVER_NAME` like every sandbox instance does, and + opens the private `openshell.supervisor.host.v1/SupervisorHost.Attach` + stream (`proto/supervisor_host.proto`) with the **supervisor token** in + `--token-file` as its bearer. The gateway mints that token with its + launch-signing key under a profile of its own (`typ` + `openshell-supervisor+jwt`, subject `openshell-supervisor:`, + no sandbox, generation, or epoch claims) and keeps the file fresh through + its deployment packaging (the Helm chart's `-supervisor-token` + Secret). The token authorizes nothing but this stream; sandbox, extension, + user, operator, and peer credentials are refused on it. The host re-reads + the file on every connection attempt and treats a missing, malformed, or + expired token as a reason to retry later, and the gateway ends a stream + when the token that opened it expires, so the host reconnects with the + renewed one. The host never acts with any other identity and has no + Kubernetes identity. The stream starts with a hello carrying a per-process + instance id and a connection epoch; the gateway supersedes an older stream + of the same process and refuses a delayed older connection. +- **Assignments.** The gateway owns the desired set and sends it as complete + `AssignmentSnapshot`s with session-scoped revisions. `shared_host::SharedHost::reconcile` + validates the snapshot as a whole (at most 256 instances, no route twice), + decodes each assignment's launch material (the JSON + `SandboxInstanceRegistration` of `openshell-supervisor-control`), admits it into + the `SandboxInstanceHost` registry keyed by route with the registration + digest as its fingerprint, and removes every hosted route the snapshot + omits. An identical assignment is a no-op; one that cannot be admitted + (invalid material, or a different fingerprint for a live route) is reported + as a failed instance and never touches the running one. Snapshots are + reconciled one at a time on a separate task, so heartbeats and status keep + flowing while instances stop; only the newest unapplied snapshot is kept, + and a connection finishes the reconciliation it started before the next + connection begins. Private files live under `--session-root` (proxy CA, + trust directory, SSH socket); the registry owns removal and keeps the route + reserved until the instance has ended and its files are deleted. +- **Status.** After each reconciliation and whenever an instance is admitted, + changes phase or readiness, or is removed, the host sends a complete + `HostStatus` naming the revision it has reconciled to; readiness is set only + once the gateway has accepted the instance's own supervisor session. +- **Disconnects.** Losing the gateway stream never stops an instance. The + host reconnects with backoff, reports what it runs, and applies the first + snapshot of the new session; only a snapshot that omits an instance, or + process shutdown, stops it. +- **Runtime connections.** One routed listener (`--runtime-listen`) accepts every + instance's runtime-initiated Sandbox Protocol connections; see + `openshell-sandbox-backend` for the routing preface. +- **Health.** `--address` is a plain TCP listener that accepts and closes + kubelet `tcpSocket` probe connections; it carries no control traffic and + reports only that the process runs. SIGTERM and SIGINT stop every instance, + which ends their workloads' supervision. + +Sandbox instances get no upstream proxy, extra CA bundle, or provider workload identity: +the Kubernetes driver refuses the topology when an upstream proxy or provider +SPIFFE identity is configured, so a workload never silently loses them. The +host also initializes no OTLP trace or OCSF exporters, because one process +cannot export each instance's records with that instance's collector headers and +identity. Its console logs and each instance's gateway log push remain, and the +Kubernetes driver warns at startup when the gateway configures supervisor +export. diff --git a/crates/openshell-supervisor/src/backend_setup.rs b/crates/openshell-supervisor/src/backend_setup.rs index fe11068b52..91d4894c09 100644 --- a/crates/openshell-supervisor/src/backend_setup.rs +++ b/crates/openshell-supervisor/src/backend_setup.rs @@ -242,6 +242,61 @@ impl BackendSetup for OpenShellBackendSetup { } } +/// The built-in Sandbox Protocol for one sandbox instance of a shared +/// supervisor. +/// +/// The descriptor must use the routed transport for exactly `route`; +/// decoding claims that route on the shared listener for as long as the +/// instance's backend exists. +pub struct RoutedBackendSetup { + router: Arc, + route: String, +} + +impl RoutedBackendSetup { + #[must_use] + pub fn new( + router: Arc, + route: String, + ) -> Self { + Self { router, route } + } +} + +impl BackendSetup for RoutedBackendSetup { + fn backend_name(&self) -> &str { + openshell_sandbox_backend::BACKEND_NAME + } + + fn decode( + &self, + payload: &[u8], + ) -> std::result::Result<(LaunchIdentity, Box), BackendError> { + let descriptor: openshell_sandbox_backend::boundary_protocol::SandboxRuntimeDescriptor = + serde_json::from_slice(payload).map_err(|error| { + BackendError::Descriptor(format!("decode sandbox runtime descriptor: {error}")) + })?; + openshell_sandbox_backend::validate_runtime_coordinates(&descriptor)?; + let registration = self.router.register(&self.route, &descriptor.tls)?; + let transport = + openshell_sandbox_backend::BoundaryTransport::routed(&descriptor, registration)?; + let identity = LaunchIdentity { + sandbox_id: descriptor.boundary_id.clone(), + generation: descriptor.generation.clone(), + session_id: descriptor.session_id, + workload_identity: descriptor.workload_identity.clone(), + vm_policy_identity: None, + }; + Ok(( + identity, + Box::new(OpenShellLaunch { + descriptor, + transport, + }), + )) + } +} + /// Launch data plus the transport chosen for it. Discovery and the attached /// backend use clones of one transport, so a runtime-initiated descriptor binds /// its control listener once for the supervisor's lifetime. diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index 39699751e2..46eb9e8839 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -25,6 +25,7 @@ mod endpoint_status; mod mechanistic_mapper; mod provider_readiness; pub mod sandbox_instance_host; +pub mod shared_host; use config_runtime::{agent_proposals_enabled_from_settings, run_policy_poll_loop}; use miette::{IntoDiagnostic, Result, WrapErr}; @@ -37,6 +38,7 @@ use std::time::Duration; use tracing::{Instrument as _, debug, info, warn}; use openshell_core::PolicyValidationFailureMode; +use openshell_supervisor_control::RegistrationTraceContext; use openshell_ocsf::{ ActionId, ActivityId, AppLifecycleBuilder, ConfidenceId, ConfigStateChangeBuilder, @@ -740,27 +742,18 @@ pub fn sandbox_instance_event_context( } } -/// W3C trace context of the operation that launched a sandbox generation. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct TraceContext { - pub traceparent: String, - pub tracestate: Option, -} - -impl TraceContext { - /// The context a driver passed to a dedicated supervisor process. - #[must_use] - pub fn from_environment() -> Option { - let traceparent = std::env::var(openshell_otel::TRACEPARENT_ENV) +/// The trace context a driver passed to a dedicated supervisor process. +#[must_use] +pub fn trace_context_from_environment() -> Option { + let traceparent = std::env::var(openshell_otel::TRACEPARENT_ENV) + .ok() + .filter(|value| !value.is_empty())?; + Some(RegistrationTraceContext { + traceparent, + tracestate: std::env::var(openshell_otel::TRACESTATE_ENV) .ok() - .filter(|value| !value.is_empty())?; - Some(Self { - traceparent, - tracestate: std::env::var(openshell_otel::TRACESTATE_ENV) - .ok() - .filter(|value| !value.is_empty()), - }) - } + .filter(|value| !value.is_empty()), + }) } /// Trusted launch inputs of one supervised sandbox instance. @@ -794,7 +787,7 @@ pub struct SandboxInstanceConfig { pub admitted_isolation_backend: Option, pub main_exit_marker: Option, /// Parent of the startup span. - pub trace_context: Option, + pub trace_context: Option, /// SPIFFE Workload API socket for provider token grants; absent fails closed. pub provider_workload_api_socket: Option, pub proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles, diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index b222c1c196..de29e9509a 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -64,10 +64,25 @@ struct Args { #[arg(long, required_if_eq("mode", "shared"))] gateway_endpoint: Option, - /// Shared supervisor listen address, also used for TCP health probes. + /// Shared supervisor TCP health listener: it accepts and closes probe + /// connections and carries no control traffic. #[arg(long, required_if_eq("mode", "shared"))] address: Option, + /// File holding the supervisor token the gateway issues for the shared + /// supervisor's assignment stream. Re-read on every connection attempt, + /// so a renewed token takes effect without a restart. + #[arg(long, required_if_eq("mode", "shared"))] + token_file: Option, + + /// Listener for routed runtime-initiated Sandbox Protocol connections. + #[arg(long, required_if_eq("mode", "shared"))] + runtime_listen: Option, + + /// Private directory for shared-host sandbox instance files. + #[arg(long, required_if_eq("mode", "shared"))] + session_root: Option, + /// Supervisor responsibility to run. #[arg(long, value_enum, default_value_t)] role: SupervisorRole, @@ -389,18 +404,16 @@ fn main() -> Result<()> { let args = Args::parse(); if args.mode == SupervisorMode::Shared { - let address = args - .address - .ok_or_else(|| miette::miette!("--address is required for --mode=shared"))?; - let gateway_endpoint = args - .gateway_endpoint - .as_deref() - .ok_or_else(|| miette::miette!("--gateway-endpoint is required for --mode=shared"))?; - return shared::run(address, gateway_endpoint); + return shared::run(args); } - if args.gateway_endpoint.is_some() || args.address.is_some() { + if args.gateway_endpoint.is_some() + || args.address.is_some() + || args.token_file.is_some() + || args.runtime_listen.is_some() + || args.session_root.is_some() + { return Err(miette::miette!( - "--gateway-endpoint and --address require --mode=shared" + "shared supervisor options require --mode=shared" )); } validate_role_arguments(&args)?; @@ -576,7 +589,7 @@ fn main() -> Result<()> { auth_bundle, admitted_isolation_backend: Some(admitted_backend), main_exit_marker: args.main_exit_marker, - trace_context: openshell_supervisor::TraceContext::from_environment(), + trace_context: openshell_supervisor::trace_context_from_environment(), provider_workload_api_socket: std::env::var_os( openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, ) diff --git a/crates/openshell-supervisor/src/sandbox_instance_host.rs b/crates/openshell-supervisor/src/sandbox_instance_host.rs index 65a8b27534..66aa30e57e 100644 --- a/crates/openshell-supervisor/src/sandbox_instance_host.rs +++ b/crates/openshell-supervisor/src/sandbox_instance_host.rs @@ -19,14 +19,16 @@ use std::collections::HashMap; use std::future::Future; +use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex, MutexGuard}; use std::time::Duration; -use miette::Result; +use miette::{IntoDiagnostic as _, Result, WrapErr as _}; use openshell_core::grpc_client::GatewayCredentials; use openshell_ocsf::OcsfJsonSettings; +use openshell_supervisor_control::{SandboxInstancePhaseReport, SandboxInstanceStatusReport}; use openshell_supervisor_network::policy_local::{DenialLogSource, SandboxInstanceDenialLog}; use openshell_supervisor_process::log_push::LogSink; use tokio::sync::watch; @@ -90,36 +92,6 @@ impl std::io::Write for SandboxInstanceDenialWriter { } } -/// Observable lifecycle of one hosted sandbox instance. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum SandboxInstancePhase { - /// Startup has not yet produced an accepted gateway session. - Starting, - /// The workload started and the gateway accepted supervision at least once. - Supervising, - /// The owner asked the instance to stop. - Stopping, - /// Supervision ended with the workload's exit status. - Exited(i32), - /// Startup or supervision failed; the workload is no longer supervised. - Failed(String), -} - -impl SandboxInstancePhase { - #[must_use] - pub fn is_terminal(&self) -> bool { - matches!(self, Self::Exited(_) | Self::Failed(_)) - } -} - -/// Current state of one hosted sandbox instance. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct SandboxInstanceStatus { - pub phase: SandboxInstancePhase, - /// Gateway supervision is currently accepted for this instance. - pub ready: bool, -} - /// Body of a hosted sandbox instance, constructed on its own runtime. pub type SandboxInstanceBody = Box< dyn FnOnce(SandboxInstanceControls) -> Pin>>> @@ -127,41 +99,84 @@ pub type SandboxInstanceBody = Box< + 'static, >; -/// The hosted sandbox instance registration was rejected. +/// Trusted identity of one sandbox instance asking to be hosted. +pub struct SandboxInstanceAdmission { + /// Unique key for this generation within the host. It also names the + /// instance's private directory, so it must be one path component. + pub key: String, + /// Digest of the launch material. Admitting the same digest under an + /// occupied key again is idempotent. + pub fingerprint: [u8; 32], + pub sandbox_id: String, +} + +/// Outcome of an accepted admission. +#[derive(Debug, PartialEq, Eq)] +pub enum SandboxInstanceAdmitted { + Created, + /// The identical admission is already hosted. + Unchanged, +} + +/// The hosted sandbox instance admission was rejected. #[derive(Debug, thiserror::Error, PartialEq, Eq)] pub enum SandboxInstanceHostError { - #[error("a sandbox instance with this key is already hosted")] - Duplicate, + #[error("sandbox instance key must be a single path component")] + InvalidKey, + #[error("key is already hosted with different launch material")] + Conflict, + #[error("key is still being removed")] + Removing, + #[error("prepare sandbox instance files: {0}")] + Files(String), #[error("start sandbox instance thread: {0}")] Thread(String), } +/// Final status of one removal, shared by every remover. +type RemovalReport = watch::Receiver>; + struct HostedSandboxInstance { /// Distinguishes this instance from a later one that reuses its key. serial: u64, + fingerprint: [u8; 32], + sandbox_id: String, stop: watch::Sender, abort: watch::Sender, - phase: watch::Receiver, + phase: watch::Receiver, ready: watch::Receiver, thread: Option>, - /// Final status of the removal in progress, shared by every remover. - removal: Option>>, + /// The removal in progress, or the report of one that could not delete + /// the instance's private files and so kept its key. + removal: Option, } impl HostedSandboxInstance { - fn status(&self) -> SandboxInstanceStatus { + /// Status of the instance hosted under `key`, which is its route. + fn status(&self, key: &str) -> SandboxInstanceStatusReport { + if let Some(reported) = self + .removal + .as_ref() + .and_then(|report| report.borrow().clone()) + { + return reported; + } let phase = self.phase.borrow().clone(); // The runner always reports a terminal phase, even while unwinding; // a closed channel without one still means supervision has ended. if !phase.is_terminal() && self.phase.has_changed().is_err() { - return SandboxInstanceStatus { - phase: SandboxInstancePhase::Failed( - "sandbox instance thread ended unexpectedly".to_string(), - ), + return SandboxInstanceStatusReport { + route: key.to_string(), + sandbox_id: self.sandbox_id.clone(), + phase: SandboxInstancePhaseReport::Failed { + error: "sandbox instance thread ended unexpectedly".to_string(), + }, ready: false, }; } - SandboxInstanceStatus { + SandboxInstanceStatusReport { + route: key.to_string(), + sandbox_id: self.sandbox_id.clone(), phase, ready: *self.ready.borrow(), } @@ -170,127 +185,199 @@ impl HostedSandboxInstance { type Instances = Arc>>; +/// Wakes observers of the registry whenever any instance's status changes. +#[derive(Clone)] +struct ChangeNotifier(Arc>); + +impl ChangeNotifier { + fn notify(&self) { + self.0.send_modify(|changes| *changes += 1); + } +} + fn lock(instances: &Instances) -> MutexGuard<'_, HashMap> { instances .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) } -/// Runs and stops sandbox instances, each on its own bound runtime. -#[derive(Default)] +/// The one registry of hosted sandbox instances. +/// +/// Each entry owns its admission fingerprint and sandbox identity, its +/// runtime thread, phase, and stop signals, and a private directory under the +/// host root. Everything else an instance owns, including its gateway +/// credentials and runtime route, lives on its runtime and ends with it. A +/// key stays reserved from admission until one removal has stopped the +/// runtime, revoked its credentials, and deleted its private directory. pub struct SandboxInstanceHost { instances: Instances, next_serial: AtomicU64, + root: PathBuf, + changes: ChangeNotifier, } impl SandboxInstanceHost { + /// Host instances whose private directories live under `root`. + /// + /// # Errors + /// + /// Fails when `root` cannot be prepared as a private directory. + pub fn new(root: PathBuf) -> Result { + prepare_private_dir(&root)?; + Ok(Self { + instances: Arc::default(), + next_serial: AtomicU64::new(0), + root, + changes: ChangeNotifier(Arc::new(watch::Sender::new(0))), + }) + } + + /// Notified whenever an instance is admitted, changes phase or readiness, + /// or is removed. The value only counts changes. #[must_use] - pub fn new() -> Self { - Self::default() + pub fn changes(&self) -> watch::Receiver { + self.changes.0.subscribe() } - /// Start `body` on a new runtime bound to `bindings`. + /// Admit a sandbox instance and start the runtime `launch` prepares in + /// the instance's new, empty private directory. /// /// The bindings' gateway credentials are revoked once the instance's /// runtime has stopped, so nothing can reuse them after the instance ends. + /// `launch` runs while the registry is locked, so concurrent admissions + /// of one key cannot both start; it must not call back into the host. /// /// # Errors /// - /// Returns [`SandboxInstanceHostError::Duplicate`] when `key` is already hosted, - /// including an instance that ended but has not been removed. - pub fn start( + /// Rejects an occupied key with different launch material or one still + /// being removed, and fails when the instance cannot be prepared or + /// started. A failed admission leaves no instance and no private files. + pub fn admit>( &self, - key: &str, - bindings: SandboxInstanceBindings, - body: SandboxInstanceBody, - ) -> std::result::Result<(), SandboxInstanceHostError> { + admission: SandboxInstanceAdmission, + launch: impl FnOnce( + &Path, + ) + -> std::result::Result<(SandboxInstanceBindings, SandboxInstanceBody), E>, + ) -> std::result::Result { + let SandboxInstanceAdmission { + key, + fingerprint, + sandbox_id, + } = admission; + if !matches!( + Path::new(&key).components().collect::>().as_slice(), + [std::path::Component::Normal(name)] if *name == key.as_str() + ) { + return Err(SandboxInstanceHostError::InvalidKey.into()); + } let mut instances = lock(&self.instances); - if instances.contains_key(key) { - return Err(SandboxInstanceHostError::Duplicate); + if let Some(existing) = instances.get(&key) { + return if existing.removal.is_some() { + Err(SandboxInstanceHostError::Removing.into()) + } else if existing.fingerprint == fingerprint { + Ok(SandboxInstanceAdmitted::Unchanged) + } else { + Err(SandboxInstanceHostError::Conflict.into()) + }; } - let (stop, stop_requested) = watch::channel(false); - let (abort, abort_requested) = watch::channel(false); - let (phase_tx, phase) = watch::channel(SandboxInstancePhase::Starting); - let (ready_tx, ready) = watch::channel(false); - let thread = std::thread::Builder::new() - .name("openshell-instance".to_string()) - .spawn(move || { - run_hosted_sandbox_instance( - &bindings, - body, - SandboxInstanceSignals { - stop_requested, - abort_requested, - phase: phase_tx, - ready: ready_tx, - }, - ); - }) - .map_err(|error| SandboxInstanceHostError::Thread(error.to_string()))?; + let dir = self.root.join(&key); + let started = prepare_instance_dir(&dir) + .map_err(|error| E::from(SandboxInstanceHostError::Files(error.to_string()))) + .and_then(|()| launch(&dir)) + .and_then(|(bindings, body)| { + spawn_instance(bindings, body, self.changes.clone()).map_err(E::from) + }); + let running = match started { + Ok(running) => running, + Err(error) => { + let _ = remove_instance_dir(&dir); + return Err(error); + } + }; instances.insert( - key.to_string(), + key, HostedSandboxInstance { serial: self.next_serial.fetch_add(1, Ordering::Relaxed), - stop, - abort, - phase, - ready, - thread: Some(thread), + fingerprint, + sandbox_id, + stop: running.stop, + abort: running.abort, + phase: running.phase, + ready: running.ready, + thread: Some(running.thread), removal: None, }, ); - Ok(()) + self.changes.notify(); + Ok(SandboxInstanceAdmitted::Created) } /// Status of a hosted sandbox instance, including one that ended but has - /// not been removed. + /// not been removed. The report's route is the instance's key. #[must_use] - pub fn status(&self, key: &str) -> Option { + pub fn status(&self, key: &str) -> Option { lock(&self.instances) .get(key) - .map(HostedSandboxInstance::status) + .map(|instance| instance.status(key)) } - /// Keys of every hosted sandbox instance. + /// Status of every hosted sandbox instance. #[must_use] - pub fn keys(&self) -> Vec { - lock(&self.instances).keys().cloned().collect() + pub fn list(&self) -> Vec { + lock(&self.instances) + .iter() + .map(|(key, instance)| instance.status(key)) + .collect() } - /// Stop one sandbox instance and forget it. Other instances keep running. + /// Stop one sandbox instance, delete its private files, and forget it. + /// Other instances keep running. /// /// A supervising instance terminates its workload gracefully within /// `grace`; one still starting is cancelled at once, because startup has /// no safe intermediate state to wind down. Removal runs on its own task, /// so it completes even when the caller stops waiting, and concurrent - /// callers share it and its result. The key stays registered until the - /// instance's thread has finished, so the same key cannot start a second - /// owner while this one is stopping. Returns the final status, or `None` - /// when `key` is not hosted. - pub async fn remove(&self, key: &str, grace: Duration) -> Option { - let (mut removed, phase) = self.begin_removal(key, grace)?; + /// callers share it and its result. The key stays reserved until the + /// instance's thread has finished and its private directory is gone; if + /// the directory cannot be deleted, the key stays reserved and the next + /// removal tries again. Returns the final status, or `None` when `key` is + /// not hosted. + pub async fn remove(&self, key: &str, grace: Duration) -> Option { + let (mut removed, sandbox_id) = self.begin_removal(key, grace)?; // The removal task ends without a report only with its runtime. - removed + let reported = removed .wait_for(Option::is_some) .await - .map_or_else(|_| Some(terminal_status(&phase)), |status| status.clone()) + .ok() + .and_then(|status| status.clone()); + Some(reported.unwrap_or_else(|| SandboxInstanceStatusReport { + route: key.to_string(), + sandbox_id, + phase: SandboxInstancePhaseReport::Failed { + error: "sandbox instance removal did not finish".to_string(), + }, + ready: false, + })) } - fn begin_removal( - &self, - key: &str, - grace: Duration, - ) -> Option<( - watch::Receiver>, - watch::Receiver, - )> { + /// Stop every sandbox instance, for process shutdown. + pub async fn remove_all(&self, grace: Duration) { + let keys = lock(&self.instances).keys().cloned().collect::>(); + futures::future::join_all(keys.iter().map(|key| self.remove(key, grace))).await; + } + + fn begin_removal(&self, key: &str, grace: Duration) -> Option<(RemovalReport, String)> { let mut instances = lock(&self.instances); let instance = instances.get_mut(key)?; - if let Some(removed) = &instance.removal { - return Some((removed.clone(), instance.phase.clone())); + // Join a removal in progress; retry one that kept the key. + if let Some(removed) = &instance.removal + && removed.borrow().is_none() + { + return Some((removed.clone(), instance.sandbox_id.clone())); } instance.stop.send_replace(true); - if *instance.phase.borrow() == SandboxInstancePhase::Starting { + if *instance.phase.borrow() == SandboxInstancePhaseReport::Starting { instance.abort.send_replace(true); } let (report, removed) = watch::channel(None); @@ -302,19 +389,15 @@ impl SandboxInstanceHost { serial: instance.serial, phase: instance.phase.clone(), thread: instance.thread.take(), + dir: self.root.join(key), grace, report, + changes: self.changes.clone(), } .run(), ); - Some((removed, instance.phase.clone())) - } -} - -fn terminal_status(phase: &watch::Receiver) -> SandboxInstanceStatus { - SandboxInstanceStatus { - phase: phase.borrow().clone(), - ready: false, + self.changes.notify(); + Some((removed, instance.sandbox_id.clone())) } } @@ -323,17 +406,19 @@ struct Removal { instances: Instances, key: String, serial: u64, - phase: watch::Receiver, + phase: watch::Receiver, thread: Option>, + dir: PathBuf, grace: Duration, - report: watch::Sender>, + report: watch::Sender>, + changes: ChangeNotifier, } impl Removal { async fn run(mut self) { let finished = tokio::time::timeout( self.grace, - self.phase.wait_for(SandboxInstancePhase::is_terminal), + self.phase.wait_for(SandboxInstancePhaseReport::is_terminal), ) .await .is_ok(); @@ -344,50 +429,120 @@ impl Removal { { instance.abort.send_replace(true); } + // Once the thread ends, the runtime, its route, and every task are + // gone and the credentials are revoked. if let Some(thread) = self.thread.take() { let _ = tokio::task::spawn_blocking(move || thread.join()).await; } - let removed = { - let mut instances = lock(&self.instances); - match instances.get(&self.key) { - Some(instance) if instance.serial == self.serial => instances.remove(&self.key), - _ => None, - } + let dir = self.dir.clone(); + let cleanup = match tokio::task::spawn_blocking(move || remove_instance_dir(&dir)).await { + Ok(result) => result.map_err(|error| error.to_string()), + Err(error) => Err(error.to_string()), }; - let status = removed.map_or_else( - || terminal_status(&self.phase), - |instance| instance.status(), - ); + let mut instances = lock(&self.instances); + let Some(instance) = instances + .get(&self.key) + .filter(|instance| instance.serial == self.serial) + else { + return; + }; + let mut status = instance.status(&self.key); + status.ready = false; + match cleanup { + Ok(()) => { + instances.remove(&self.key); + } + Err(error) => { + tracing::warn!(%error, "could not delete removed sandbox instance files; keeping its key reserved"); + status.phase = SandboxInstancePhaseReport::Failed { + error: format!( + "sandbox instance stopped, but its private files remain: {error}" + ), + }; + } + } self.report.send_replace(Some(status)); + self.changes.notify(); } } +/// Signals and thread of one started instance runtime. +struct RunningInstance { + stop: watch::Sender, + abort: watch::Sender, + phase: watch::Receiver, + ready: watch::Receiver, + thread: std::thread::JoinHandle<()>, +} + +fn spawn_instance( + bindings: SandboxInstanceBindings, + body: SandboxInstanceBody, + changes: ChangeNotifier, +) -> std::result::Result { + let (stop, stop_requested) = watch::channel(false); + let (abort, abort_requested) = watch::channel(false); + let (phase_tx, phase) = watch::channel(SandboxInstancePhaseReport::Starting); + let (ready_tx, ready) = watch::channel(false); + let credentials = bindings.gateway_credentials.clone(); + let thread = std::thread::Builder::new() + .name("openshell-instance".to_string()) + .spawn(move || { + run_hosted_sandbox_instance( + &bindings, + body, + SandboxInstanceSignals { + stop_requested, + abort_requested, + phase: phase_tx, + ready: ready_tx, + changes, + }, + ); + }) + .map_err(|error| { + credentials.revoke(); + SandboxInstanceHostError::Thread(error.to_string()) + })?; + Ok(RunningInstance { + stop, + abort, + phase, + ready, + thread, + }) +} + struct SandboxInstanceSignals { stop_requested: watch::Receiver, abort_requested: watch::Receiver, - phase: watch::Sender, + phase: watch::Sender, ready: watch::Sender, + changes: ChangeNotifier, } /// Reports a hosted sandbox instance's end and revokes its credentials, /// including when the instance thread unwinds. struct SandboxInstanceExit { credentials: GatewayCredentials, - phase: watch::Sender, + phase: watch::Sender, ready: watch::Sender, - outcome: Option, + outcome: Option, + changes: ChangeNotifier, } impl Drop for SandboxInstanceExit { fn drop(&mut self) { self.credentials.revoke(); self.ready.send_replace(false); - let outcome = self.outcome.take().unwrap_or_else(|| { - SandboxInstancePhase::Failed( - "sandbox instance thread ended without reporting a result".to_string(), - ) - }); + let outcome = self + .outcome + .take() + .unwrap_or_else(|| SandboxInstancePhaseReport::Failed { + error: "sandbox instance thread ended without reporting a result".to_string(), + }); self.phase.send_replace(outcome); + self.changes.notify(); unbind_thread(); } } @@ -402,12 +557,14 @@ fn run_hosted_sandbox_instance( mut abort_requested, phase, ready, + changes, } = signals; let mut exit = SandboxInstanceExit { credentials: bindings.gateway_credentials.clone(), phase: phase.clone(), ready: ready.clone(), outcome: None, + changes: changes.clone(), }; bind_thread(bindings); let thread_bindings = bindings.clone(); @@ -425,30 +582,38 @@ fn run_hosted_sandbox_instance( let (readiness, mut accepted) = watch::channel(false); let mirror_phase = phase.clone(); let mirror_ready = ready.clone(); + let mirror_changes = changes.clone(); let mirror = tokio::spawn(async move { loop { let is_ready = *accepted.borrow_and_update(); - mirror_ready.send_replace(is_ready); - if is_ready { - mirror_phase.send_if_modified(|current| { - let promote = *current == SandboxInstancePhase::Starting; + let changed = mirror_ready.send_replace(is_ready) != is_ready; + let promoted = is_ready + && mirror_phase.send_if_modified(|current| { + let promote = + *current == SandboxInstancePhaseReport::Starting; if promote { - *current = SandboxInstancePhase::Supervising; + *current = SandboxInstancePhaseReport::Supervising; } promote }); + if changed || promoted { + mirror_changes.notify(); } if accepted.changed().await.is_err() { - mirror_ready.send_replace(false); + if mirror_ready.send_replace(false) { + mirror_changes.notify(); + } return; } } }); let mut stop = stop_requested.clone(); let stop_phase = phase.clone(); + let stop_changes = changes.clone(); let shutdown = Box::pin(async move { let _ = stop.wait_for(|requested| *requested).await; - stop_phase.send_replace(SandboxInstancePhase::Stopping); + stop_phase.send_replace(SandboxInstancePhaseReport::Stopping); + stop_changes.notify(); }); let instance = body(SandboxInstanceControls { shutdown, @@ -470,85 +635,114 @@ fn run_hosted_sandbox_instance( Err(error) => Err(miette::miette!("build sandbox instance runtime: {error}")), }; exit.outcome = Some(match outcome { - Ok(code) => SandboxInstancePhase::Exited(code), - Err(error) => SandboxInstancePhase::Failed(error.to_string()), + Ok(exit_code) => SandboxInstancePhaseReport::Exited { exit_code }, + Err(error) => SandboxInstancePhaseReport::Failed { + error: error.to_string(), + }, }); } -/// Trusted inputs for one sandbox instance hosted with others. -pub struct SandboxInstanceSpec { - /// Unique key for this generation within the host. - pub key: String, - pub backend_setup: Box, - pub config: crate::SandboxInstanceConfig, +/// Prepare a hosted sandbox instance of `config` whose private files live in +/// `dir`: bindings with its own gateway credentials, log push stream, and +/// denial log, and a body that supervises it. +/// +/// # Errors +/// +/// Fails when the launch credentials are invalid. +pub fn sandbox_instance_launch( + backend_setup: Box, + mut config: crate::SandboxInstanceConfig, + dir: &Path, +) -> Result<(SandboxInstanceBindings, SandboxInstanceBody)> { + // Instances must never share proxy trust files. + config + .proxy_tls + .directory + .get_or_insert_with(|| dir.to_path_buf()); + // Log push, supervision, and renewal share these credentials. + let gateway_credentials = GatewayCredentials::from_auth_bundle(&config.auth_bundle)?; + let log_push = config + .sandbox_id + .clone() + .zip(config.openshell_endpoint.as_deref().map(|endpoint| { + openshell_core::grpc_client::GatewayEndpoint::new(endpoint, gateway_credentials.clone()) + })); + let (log_tx, log_rx) = + tokio::sync::mpsc::channel(openshell_supervisor_process::log_push::LOG_PUSH_QUEUE_LINES); + let bindings = SandboxInstanceBindings { + gateway_credentials, + event_context: Arc::new(config.event_context()), + ocsf_json: OcsfJsonSettings::default(), + log_sink: log_push + .as_ref() + .map(|(sandbox_id, _)| LogSink::new(sandbox_id.as_str(), log_tx)), + denial_log: DenialLogSource::SandboxInstance(SandboxInstanceDenialLog::default()), + }; + let instance_bindings = bindings.clone(); + let body: SandboxInstanceBody = Box::new(move |controls| { + Box::pin(async move { + let _log_push = log_push.map(|(sandbox_id, gateway)| { + openshell_supervisor_process::log_push::start_log_push(gateway, sandbox_id, log_rx) + }); + Box::pin(crate::run_sandbox_instance( + backend_setup.as_ref(), + config, + instance_bindings, + controls, + )) + .await + }) + }); + Ok((bindings, body)) } -impl SandboxInstanceHost { - /// Start a sandbox instance with its own OCSF identity, OCSF JSON - /// settings, log push stream, denial log, and gateway credentials. - /// - /// # Errors - /// - /// Fails when the launch credentials are invalid or the key is hosted. - pub fn start_sandbox_instance(&self, spec: SandboxInstanceSpec) -> Result<()> { - let SandboxInstanceSpec { - key, - backend_setup, - mut config, - } = spec; - // Instances must never share proxy trust files; give one a private - // directory that lives exactly as long as the instance. - let private_tls_dir = if config.proxy_tls.directory.is_none() { - let prepared = crate::prepare_network_proxy_tls_dir(None)?; - config.proxy_tls.directory = Some(prepared.path.clone()); - Some(prepared) - } else { - None - }; - // Log push, supervision, and renewal share these credentials. - let gateway_credentials = GatewayCredentials::from_auth_bundle(&config.auth_bundle)?; - let log_push = config - .sandbox_id - .clone() - .zip(config.openshell_endpoint.as_deref().map(|endpoint| { - openshell_core::grpc_client::GatewayEndpoint::new( - endpoint, - gateway_credentials.clone(), - ) - })); - let (log_tx, log_rx) = tokio::sync::mpsc::channel( - openshell_supervisor_process::log_push::LOG_PUSH_QUEUE_LINES, - ); - let bindings = SandboxInstanceBindings { - gateway_credentials, - event_context: Arc::new(config.event_context()), - ocsf_json: OcsfJsonSettings::default(), - log_sink: log_push - .as_ref() - .map(|(sandbox_id, _)| LogSink::new(sandbox_id.as_str(), log_tx)), - denial_log: DenialLogSource::SandboxInstance(SandboxInstanceDenialLog::default()), - }; - let instance_bindings = bindings.clone(); - let body: SandboxInstanceBody = Box::new(move |controls| { - Box::pin(async move { - let _private_tls_dir = private_tls_dir; - let _log_push = log_push.map(|(sandbox_id, gateway)| { - openshell_supervisor_process::log_push::start_log_push( - gateway, sandbox_id, log_rx, - ) - }); - Box::pin(crate::run_sandbox_instance( - backend_setup.as_ref(), - config, - instance_bindings, - controls, - )) - .await - }) - }); - self.start(&key, bindings, body) - .map_err(|error| miette::miette!("{error}")) +/// Create `dir` as an empty private directory, replacing files a previous +/// process left behind. +fn prepare_instance_dir(dir: &Path) -> Result<()> { + remove_instance_dir(dir) + .into_diagnostic() + .wrap_err_with(|| format!("delete stale sandbox instance files {}", dir.display()))?; + prepare_private_dir(dir) +} + +fn remove_instance_dir(dir: &Path) -> std::io::Result<()> { + match std::fs::remove_dir_all(dir) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + result => result, + } +} + +#[cfg(unix)] +pub(crate) fn prepare_private_dir(path: &Path) -> Result<()> { + use std::os::unix::fs::{DirBuilderExt as _, PermissionsExt as _}; + + match std::fs::DirBuilder::new().mode(0o700).create(path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + let metadata = std::fs::symlink_metadata(path).into_diagnostic()?; + if !metadata.is_dir() { + return Err(miette::miette!( + "sandbox instance path is not a directory: {}", + path.display() + )); + } + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)) + .into_diagnostic()?; + } + Err(error) => { + return Err(error) + .into_diagnostic() + .wrap_err_with(|| format!("create private directory {}", path.display())); + } } + Ok(()) +} + +#[cfg(not(unix))] +pub(crate) fn prepare_private_dir(_path: &Path) -> Result<()> { + Err(miette::miette!( + "hosted sandbox instances require a Unix host" + )) } #[cfg(test)] @@ -700,7 +894,10 @@ mod tests { } }) }); - host.start(sandbox, bindings, body).unwrap(); + assert_eq!( + admit(host, sandbox, sandbox, 0, bindings, body), + Ok(SandboxInstanceAdmitted::Created) + ); Instance { identity, logs, @@ -713,7 +910,7 @@ mod tests { async fn wait_supervising(host: &SandboxInstanceHost, key: &str) { tokio::time::timeout(Duration::from_secs(10), async { while host.status(key).map(|status| status.phase) - != Some(SandboxInstancePhase::Supervising) + != Some(SandboxInstancePhaseReport::Supervising) { tokio::time::sleep(Duration::from_millis(10)).await; } @@ -736,42 +933,154 @@ mod tests { } } + /// Admit `body` under `key`, recording `launch` in its private directory. + fn admit( + host: &SandboxInstanceHost, + key: &str, + sandbox_id: &str, + launch: u8, + bindings: SandboxInstanceBindings, + body: SandboxInstanceBody, + ) -> std::result::Result { + host.admit( + SandboxInstanceAdmission { + key: key.to_string(), + fingerprint: [launch; 32], + sandbox_id: sandbox_id.to_string(), + }, + |dir| { + std::fs::write(dir.join("launch"), [launch]) + .map_err(|error| SandboxInstanceHostError::Files(error.to_string()))?; + Ok((bindings, body)) + }, + ) + } + + fn private_host() -> (tempfile::TempDir, Arc) { + let root = tempfile::tempdir().unwrap(); + let host = SandboxInstanceHost::new(root.path().join("instances")).unwrap(); + (root, Arc::new(host)) + } + #[tokio::test(flavor = "multi_thread")] - async fn stopping_and_panicking_instances_keep_their_key_and_fail_closed() { - let host = Arc::new(SandboxInstanceHost::new()); - let body: SandboxInstanceBody = Box::new(|controls| { - Box::pin(async move { - controls.readiness.send_replace(true); - controls.shutdown.await; - // Winding down takes time; the key must stay taken meanwhile. - tokio::time::sleep(Duration::from_millis(300)).await; - Ok(0) + async fn one_removal_holds_an_idempotent_admission_until_its_files_are_gone() { + let (root, host) = private_host(); + let dir = root.path().join("instances").join("generation"); + let winding_down = |code| -> SandboxInstanceBody { + Box::new(move |controls| { + Box::pin(async move { + controls.readiness.send_replace(true); + controls.shutdown.await; + // Winding down takes time; the key must stay taken meanwhile. + tokio::time::sleep(Duration::from_millis(300)).await; + Ok(code) + }) }) - }); - host.start("generation", unbound(), body).unwrap(); + }; + let admit_launch = |launch, code| { + admit( + &host, + "generation", + &format!("sandbox-{launch}"), + launch, + unbound(), + winding_down(code), + ) + }; + assert_eq!(admit_launch(1, 3), Ok(SandboxInstanceAdmitted::Created)); + assert_eq!(admit_launch(1, 3), Ok(SandboxInstanceAdmitted::Unchanged)); + assert_eq!(admit_launch(2, 3), Err(SandboxInstanceHostError::Conflict)); + assert_eq!( + admit( + &host, + "../escape", + "sandbox-1", + 1, + unbound(), + winding_down(0) + ), + Err(SandboxInstanceHostError::InvalidKey) + ); wait_supervising(&host, "generation").await; - let removal = tokio::spawn({ - let host = host.clone(); - async move { host.remove("generation", Duration::from_secs(10)).await } - }); + + // A caller that gives up, such as an expired RPC, strands nothing. + assert!( + tokio::time::timeout( + Duration::from_millis(20), + host.remove("generation", Duration::from_secs(10)) + ) + .await + .is_err() + ); tokio::time::timeout(Duration::from_secs(10), async { while host.status("generation").map(|status| status.phase) - != Some(SandboxInstancePhase::Stopping) + != Some(SandboxInstancePhaseReport::Stopping) { tokio::time::sleep(Duration::from_millis(5)).await; } }) .await .expect("instance observes the stop request"); - let replacement: SandboxInstanceBody = Box::new(|_| Box::pin(async { Ok(0) })); + assert_eq!(admit_launch(1, 3), Err(SandboxInstanceHostError::Removing)); + assert!(dir.join("launch").exists()); + let removers = [(); 2].map(|()| { + let host = host.clone(); + tokio::spawn(async move { host.remove("generation", Duration::from_secs(10)).await }) + }); + for remover in removers { + let status = remover.await.unwrap().expect("instance was hosted"); + assert_eq!(status.sandbox_id, "sandbox-1"); + assert_eq!( + status.phase, + SandboxInstancePhaseReport::Exited { exit_code: 3 } + ); + } + assert!(host.status("generation").is_none()); + assert!(!dir.exists()); + + // The key is free again, and the earlier removal cannot touch its new owner. + assert_eq!(admit_launch(4, 4), Ok(SandboxInstanceAdmitted::Created)); + wait_supervising(&host, "generation").await; + tokio::time::sleep(Duration::from_millis(50)).await; assert_eq!( - host.start("generation", unbound(), replacement), - Err(SandboxInstanceHostError::Duplicate) + host.status("generation").map(|status| status.phase), + Some(SandboxInstancePhaseReport::Supervising) ); - let stopped = removal.await.unwrap().expect("instance was hosted"); - assert_eq!(stopped.phase, SandboxInstancePhase::Exited(0)); - assert!(host.status("generation").is_none()); + assert_eq!(std::fs::read(dir.join("launch")).unwrap(), [4]); + + // Files that cannot be deleted keep the key reserved until a later + // removal deletes them. Root ignores the permission this relies on. + #[cfg(unix)] + if !nix::unistd::geteuid().is_root() { + use std::os::unix::fs::PermissionsExt as _; + let instances = root.path().join("instances"); + std::fs::set_permissions(&instances, std::fs::Permissions::from_mode(0o500)).unwrap(); + let status = host + .remove("generation", Duration::from_secs(10)) + .await + .unwrap(); + assert!( + matches!(&status.phase, SandboxInstancePhaseReport::Failed { error } if error.contains("private files remain")), + "{status:?}" + ); + assert_eq!(host.status("generation"), Some(status)); + assert_eq!(admit_launch(5, 5), Err(SandboxInstanceHostError::Removing)); + std::fs::set_permissions(&instances, std::fs::Permissions::from_mode(0o700)).unwrap(); + } + let status = host.remove("generation", Duration::from_secs(10)).await; + assert_eq!( + status.map(|status| status.phase), + Some(SandboxInstancePhaseReport::Exited { exit_code: 4 }) + ); + assert!(!dir.exists()); + host.remove_all(Duration::from_secs(10)).await; + assert!(host.list().is_empty()); + } + + #[tokio::test(flavor = "multi_thread")] + async fn panicking_instance_fails_closed_and_revokes_its_credentials() { + let (_root, host) = private_host(); let credentials = GatewayCredentials::from_gateway_token("panicking-token").unwrap(); let panicking: SandboxInstanceBody = Box::new(|controls| { Box::pin(async move { @@ -780,8 +1089,11 @@ mod tests { panic!("hosted sandbox instance failure"); }) }); - host.start( + admit( + &host, "panicking", + "sandbox-panicking", + 1, SandboxInstanceBindings { gateway_credentials: credentials.clone(), ..unbound() @@ -803,8 +1115,12 @@ mod tests { .expect("panicking instance reports an end"); assert_eq!( status, - SandboxInstanceStatus { - phase: SandboxInstancePhase::Failed("sandbox instance panicked".to_string()), + SandboxInstanceStatusReport { + route: "panicking".to_string(), + sandbox_id: "sandbox-panicking".to_string(), + phase: SandboxInstancePhaseReport::Failed { + error: "sandbox instance panicked".to_string(), + }, ready: false, } ); @@ -820,73 +1136,25 @@ mod tests { ); } - #[tokio::test(flavor = "multi_thread")] - async fn removal_outlives_its_caller_and_reports_to_every_remover() { - let host = Arc::new(SandboxInstanceHost::new()); - let winding_down = |code| -> SandboxInstanceBody { - Box::new(move |controls| { - Box::pin(async move { - controls.readiness.send_replace(true); - controls.shutdown.await; - tokio::time::sleep(Duration::from_millis(300)).await; - Ok(code) - }) - }) - }; - host.start("generation", unbound(), winding_down(3)) - .unwrap(); - wait_supervising(&host, "generation").await; - // A caller that gives up, such as an expired RPC, strands nothing. - assert!( - tokio::time::timeout( - Duration::from_millis(20), - host.remove("generation", Duration::from_secs(10)) - ) - .await - .is_err() - ); - let removers = [(); 2].map(|()| { - let host = host.clone(); - tokio::spawn(async move { host.remove("generation", Duration::from_secs(10)).await }) - }); - for remover in removers { - let status = remover.await.unwrap().expect("instance was hosted"); - assert_eq!(status.phase, SandboxInstancePhase::Exited(3)); - } - assert!(host.status("generation").is_none()); - - // The key is free again, and the earlier removal cannot touch its new owner. - host.start("generation", unbound(), winding_down(4)) - .unwrap(); - wait_supervising(&host, "generation").await; - tokio::time::sleep(Duration::from_millis(50)).await; - assert_eq!( - host.status("generation").map(|status| status.phase), - Some(SandboxInstancePhase::Supervising) - ); - let status = host.remove("generation", Duration::from_secs(10)).await; - assert_eq!( - status.map(|status| status.phase), - Some(SandboxInstancePhase::Exited(4)) - ); - } - #[tokio::test(flavor = "multi_thread")] async fn hosted_instances_keep_identity_credentials_and_lifecycle_separate() { let (gateway_url, captured) = capture_gateway().await; - let host = SandboxInstanceHost::new(); + let (_root, host) = private_host(); let mut alpha = start_instance(&host, &gateway_url, "sandbox-alpha"); let mut beta = start_instance(&host, &gateway_url, "sandbox-beta"); wait_supervising(&host, "sandbox-alpha").await; wait_supervising(&host, "sandbox-beta").await; - assert!(matches!( - host.start( + assert_eq!( + admit( + &host, + "sandbox-alpha", "sandbox-alpha", + 1, unbound(), Box::new(|_| Box::pin(async { Ok(0) })), ), - Err(SandboxInstanceHostError::Duplicate) - )); + Err(SandboxInstanceHostError::Conflict) + ); assert_eq!((&mut alpha.identity).await.unwrap(), "sandbox-alpha"); assert_eq!((&mut beta.identity).await.unwrap(), "sandbox-beta"); @@ -913,7 +1181,10 @@ mod tests { .remove("sandbox-alpha", Duration::from_secs(10)) .await .expect("alpha was hosted"); - assert_eq!(stopped.phase, SandboxInstancePhase::Exited(0)); + assert_eq!( + stopped.phase, + SandboxInstancePhaseReport::Exited { exit_code: 0 } + ); assert!(!stopped.ready); assert!(host.status("sandbox-alpha").is_none()); assert_eq!(alpha.alive.recv().await, None, "alpha's tasks stopped"); @@ -937,7 +1208,7 @@ mod tests { ); assert_eq!( host.status("sandbox-beta").map(|status| status.phase), - Some(SandboxInstancePhase::Supervising) + Some(SandboxInstancePhaseReport::Supervising) ); assert!( beta.alive @@ -948,7 +1219,7 @@ mod tests { let stopped = host.remove("sandbox-beta", Duration::from_secs(10)).await; assert_eq!( stopped.map(|status| status.phase), - Some(SandboxInstancePhase::Exited(0)) + Some(SandboxInstancePhaseReport::Exited { exit_code: 0 }) ); } } diff --git a/crates/openshell-supervisor/src/shared.rs b/crates/openshell-supervisor/src/shared.rs index bc67386fd4..4dc3e521ef 100644 --- a/crates/openshell-supervisor/src/shared.rs +++ b/crates/openshell-supervisor/src/shared.rs @@ -1,58 +1,152 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Shared-host placeholder: no sandbox admission, credentials, or session RPCs. +//! Supervisor host shared by the sandboxes of one gateway. -#[cfg(unix)] -use miette::IntoDiagnostic; use miette::Result; -use std::net::SocketAddr; +#[cfg(unix)] +use std::future::Future; -// The gateway endpoint is reserved for the shared supervisor's outbound session. -// This plumbing stage only starts the local listener. -pub fn run(address: SocketAddr, _gateway_endpoint: &str) -> Result<()> { - run_platform(address) +pub fn run(args: crate::Args) -> Result<()> { + run_platform(args) } #[cfg(unix)] -fn run_platform(address: SocketAddr) -> Result<()> { - let runtime = tokio::runtime::Builder::new_current_thread() +fn run_platform(args: crate::Args) -> Result<()> { + use miette::{IntoDiagnostic as _, WrapErr as _}; + use std::sync::Arc; + use std::sync::atomic::AtomicBool; + use tracing_subscriber::{Layer as _, layer::SubscriberExt as _, util::SubscriberInitExt as _}; + + let address = args + .address + .ok_or_else(|| miette::miette!("--address is required for --mode=shared"))?; + let gateway_endpoint = args + .gateway_endpoint + .ok_or_else(|| miette::miette!("--gateway-endpoint is required for --mode=shared"))?; + let token_file = args + .token_file + .ok_or_else(|| miette::miette!("--token-file is required for --mode=shared"))?; + let runtime_listen = args + .runtime_listen + .ok_or_else(|| miette::miette!("--runtime-listen is required for --mode=shared"))?; + // `--session-root` keeps its established flag name. + let instance_root = args + .session_root + .ok_or_else(|| miette::miette!("--session-root is required for --mode=shared"))?; + // The supervisor token travels only over TLS; the host verifies the + // gateway the way every sandbox instance does, with `OPENSHELL_TLS_CA` + // and `OPENSHELL_GATEWAY_TLS_SERVER_NAME`. + if !gateway_endpoint.starts_with("https://") { + return Err(miette::miette!( + "--gateway-endpoint must be https for --mode=shared" + )); + } + if std::env::var_os(openshell_core::sandbox_env::TLS_CA).is_none() { + return Err(miette::miette!( + "{} is required for --mode=shared", + openshell_core::sandbox_env::TLS_CA + )); + } + let runtime = tokio::runtime::Builder::new_multi_thread() .enable_all() .build() .into_diagnostic()?; runtime.block_on(async { - // Install shutdown handlers before publishing health. + openshell_crypto::tls::ensure_default_provider(); + let console_filter = tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new(&args.log_level)); + // Process-wide OCSF JSON output stays off; each sandbox instance's own + // settings apply to the events its threads emit. + let (logging_layers, _logging_guards, _) = crate::logging::layers( + std::io::stderr(), + std::path::Path::new("/var/log"), + console_filter, + Arc::new(AtomicBool::new(false)), + Arc::new(std::sync::Mutex::new(String::new())), + ); + tracing_subscriber::registry() + .with(logging_layers) + .with(openshell_supervisor_process::log_push::LogPushLayer::thread_bound()) + .with( + openshell_ocsf::OcsfShorthandLayer::new( + openshell_supervisor::sandbox_instance_host::SandboxInstanceDenialWriter, + ) + .with_non_ocsf(false) + .with_filter(tracing_subscriber::filter::LevelFilter::INFO), + ) + .init(); + + // Install shutdown handlers before the health listener starts + // answering probes. let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) .into_diagnostic()?; let mut interrupt = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt()) .into_diagnostic()?; - let listener = tokio::net::TcpListener::bind(address) + let link = openshell_supervisor::shared_host::GatewayLink { + endpoint: gateway_endpoint.clone(), + token_file, + }; + let host = openshell_supervisor::shared_host::SharedHost::start( + openshell_supervisor::shared_host::SharedHostConfig { + gateway_endpoint, + instance_root, + runtime_listen, + removal_grace: openshell_supervisor_control::INSTANCE_REMOVAL_GRACE, + }, + )?; + let health = tokio::net::TcpListener::bind(address) .await - .into_diagnostic()?; - eprintln!( - "Shared supervisor started; gateway connection and sandbox sessions are not implemented" + .into_diagnostic() + .wrap_err("bind health listener")?; + tracing::info!( + runtime = %host.runtime_addr(), + health = %address, + "Shared supervisor ready for sandbox instance assignments" ); - loop { - tokio::select! { - // This placeholder only accepts and closes connections, including - // health probes. Session control will use an outbound gateway stream. - result = listener.accept() => { - if let Err(error) = result { - eprintln!("shared supervisor listener accept failed: {error}"); - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - } - } - _ = terminate.recv() => break, - _ = interrupt.recv() => break, - } + let (stop, stopped) = tokio::sync::watch::channel(false); + let wait_stopped = |mut stopped: tokio::sync::watch::Receiver| async move { + let _ = stopped.wait_for(|stop| *stop).await; + }; + let probes = tokio::spawn(serve_health(health, wait_stopped(stopped.clone()))); + let attachment = tokio::spawn(openshell_supervisor::shared_host::attach_to_gateway( + host.clone(), + link, + wait_stopped(stopped), + )); + tokio::select! { + _ = terminate.recv() => {} + _ = interrupt.recv() => {} } + stop.send_replace(true); + let _ = probes.await; + let _ = attachment.await; + // Sandboxes lose supervision with this process and fail closed. + host.remove_all().await; Ok(()) }) } +/// Answer kubelet `tcpSocket` probes: accept and close. Liveness means the +/// process runs; sandbox instance readiness is reported to the gateway. +#[cfg(unix)] +async fn serve_health(listener: tokio::net::TcpListener, shutdown: impl Future) { + tokio::pin!(shutdown); + loop { + tokio::select! { + () = &mut shutdown => break, + accepted = listener.accept() => { + if accepted.is_err() { + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + } + } + } +} + #[cfg(not(unix))] -fn run_platform(_address: SocketAddr) -> Result<()> { +fn run_platform(_args: crate::Args) -> Result<()> { Err(miette::miette!("--mode=shared requires Unix")) } diff --git a/crates/openshell-supervisor/src/shared_host.rs b/crates/openshell-supervisor/src/shared_host.rs new file mode 100644 index 0000000000..b78ce7469c --- /dev/null +++ b/crates/openshell-supervisor/src/shared_host.rs @@ -0,0 +1,724 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Supervisor shared by the sandboxes of one gateway. +//! +//! The gateway owns which sandbox instances, one running generation of a +//! stable Sandbox resource each, this process hosts. The host dials the +//! gateway presenting the supervisor token the gateway minted for it, +//! receives complete assignment snapshots over the private +//! `openshell.supervisor.host.v1` stream, reconciles the +//! [`SandboxInstanceHost`] registry to each one, and reports what it +//! observes. An assignment carries exactly the gateway-issued credentials of +//! its generation; the host never acts with any other identity toward +//! sandbox APIs. Losing the gateway stream never stops an instance; +//! only a later snapshot that omits it does. Snapshots are reconciled one at +//! a time, and a connection finishes the reconciliation it started before the +//! next connection begins, so a superseded stream can never apply after its +//! replacement. Runtime connections arrive through one routed listener. + +use std::collections::BTreeMap; +use std::future::Future; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use miette::Result; +use openshell_core::proto::supervisor_host::v1::supervisor_host_client::SupervisorHostClient; +use openshell_core::proto::supervisor_host::v1::{ + AssignmentSnapshot, HostHeartbeat, HostHello, HostMessage, HostStatus, host_gateway_message, + host_message, +}; +use openshell_extension_core::BearerTokenSlot; +use openshell_sandbox_backend::runtime_routed::RuntimeRouter; +use openshell_supervisor_control::{ + SandboxInstancePhaseReport, SandboxInstanceRegistration, SandboxInstanceStatusReport, +}; +use tokio::sync::watch; +use tokio_stream::StreamExt as _; + +use crate::sandbox_instance_host::{ + SandboxInstanceAdmission, SandboxInstanceAdmitted, SandboxInstanceBody, SandboxInstanceHost, + SandboxInstanceHostError, +}; + +/// Interval at which the host sends heartbeats on an idle stream. +const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(15); +/// Silence after which the host abandons a stream and reconnects. +const GATEWAY_SILENCE_TIMEOUT: Duration = Duration::from_mins(1); +/// Bound on dialing the gateway, on the gateway answering the attach call, +/// and on it accepting the host. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); +/// Bound on queueing one host-to-gateway message; a gateway that cannot +/// drain the stream this long is treated as lost. +const SEND_TIMEOUT: Duration = Duration::from_secs(30); +const RECONNECT_BACKOFF_MIN: Duration = Duration::from_secs(1); +const RECONNECT_BACKOFF_MAX: Duration = Duration::from_secs(30); +/// Host-to-gateway messages queued before the stream applies backpressure. +const OUTBOUND_QUEUE: usize = 16; +/// Most sandbox instances one snapshot may assign to a host; the gateway +/// enforces the same bound before it assigns. +pub const MAX_ASSIGNED_INSTANCES: usize = openshell_supervisor_control::MAX_ASSIGNED_INSTANCES; +/// Largest gateway message the host decodes: a full snapshot of +/// `MAX_ASSIGNED_INSTANCES` assignments at their largest, plus framing. +const MAX_GATEWAY_MESSAGE_BYTES: usize = + MAX_ASSIGNED_INSTANCES * (openshell_supervisor_control::MAX_LAUNCH_MATERIAL_BYTES + 1024); +/// Largest host message: a status report is a few hundred bytes per instance. +const MAX_HOST_MESSAGE_BYTES: usize = 1024 * 1024; + +/// Process-level inputs of a shared supervisor. +pub struct SharedHostConfig { + /// Gateway gRPC endpoint every sandbox instance uses with its own + /// credentials. + pub gateway_endpoint: String, + /// Private directory for per-instance files (`--session-root`). + pub instance_root: PathBuf, + /// Listener for runtime-initiated Sandbox Protocol connections. + pub runtime_listen: SocketAddr, + /// Time a withdrawn sandbox instance may take to stop its workload. + pub removal_grace: Duration, +} + +/// The sandbox instance registry and the routed runtime listener its +/// instances claim routes on, reconciled to the gateway's assignments. +pub struct SharedHost { + host: SandboxInstanceHost, + router: Arc, + gateway_endpoint: String, + removal_grace: Duration, + /// Assignments of the latest snapshot that could not be admitted, reported + /// as failed instances until the gateway withdraws them. + rejected: Mutex>, + /// Routes of the latest applied snapshot, so a withdrawal whose cleanup + /// failed can be retried without the gateway resending the snapshot. + desired_routes: Mutex>>, +} + +/// A rejected sandbox instance assignment. +#[derive(Debug, thiserror::Error)] +pub enum ControlError { + #[error("{0}")] + Invalid(String), + #[error("route is already hosted with different launch material")] + Conflict, + #[error("route is still being removed")] + Removing, + #[error("{0}")] + Unavailable(String), +} + +impl From for ControlError { + fn from(error: SandboxInstanceHostError) -> Self { + match error { + SandboxInstanceHostError::InvalidKey => Self::Invalid(error.to_string()), + SandboxInstanceHostError::Conflict => Self::Conflict, + SandboxInstanceHostError::Removing => Self::Removing, + SandboxInstanceHostError::Files(_) | SandboxInstanceHostError::Thread(_) => { + Self::Unavailable(error.to_string()) + } + } + } +} + +impl SharedHost { + /// Bind the routed runtime listener and prepare the instance root. + pub fn start(config: SharedHostConfig) -> Result> { + let host = SandboxInstanceHost::new(config.instance_root)?; + let router = RuntimeRouter::bind(config.runtime_listen) + .map_err(|error| miette::miette!("{error}"))?; + Ok(Arc::new(Self { + host, + router, + gateway_endpoint: config.gateway_endpoint, + removal_grace: config.removal_grace, + rejected: Mutex::new(BTreeMap::new()), + desired_routes: Mutex::new(None), + })) + } + + /// Address of the routed runtime listener. + #[must_use] + pub fn runtime_addr(&self) -> SocketAddr { + self.router.local_addr() + } + + /// Start a sandbox instance for `registration`, or confirm an identical + /// one. + pub fn register( + &self, + registration: SandboxInstanceRegistration, + ) -> Result { + registration + .validate() + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let admission = SandboxInstanceAdmission { + key: registration.route.clone(), + fingerprint: registration_digest(®istration)?, + sandbox_id: registration.sandbox_id.clone(), + }; + self.host + .admit(admission, |dir| self.instance_launch(registration, dir)) + } + + /// Status of one hosted sandbox instance. + #[must_use] + pub fn status(&self, route: &str) -> Option { + self.host.status(route) + } + + /// Status of every hosted sandbox instance. An assignment of the latest + /// snapshot that could not be admitted is reported as failed, even when + /// an earlier instance still runs under its route, so the gateway sees + /// the rejection rather than the instance it could not replace. + #[must_use] + pub fn list(&self) -> Vec { + let rejected = self + .rejected + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let mut reports = self + .host + .list() + .into_iter() + .filter(|hosted| !rejected.contains_key(&hosted.route)) + .collect::>(); + reports.extend(rejected.values().cloned()); + reports + } + + /// Notified whenever a hosted instance is admitted, changes phase or + /// readiness, or is removed. + #[must_use] + pub fn changes(&self) -> watch::Receiver { + self.host.changes() + } + + /// Stop one sandbox instance, delete its private files, and forget it. + pub async fn remove(&self, route: &str) -> Option { + self.host.remove(route, self.removal_grace).await + } + + /// Stop every sandbox instance, for process shutdown. + pub async fn remove_all(&self) { + self.host.remove_all(self.removal_grace).await; + } + + /// Make the registry match `snapshot`: admit every assignment and remove + /// every hosted instance it omits. Assignments that cannot be admitted are + /// reported as failed instances; they never affect a running instance. + /// + /// # Errors + /// + /// A malformed snapshot (too many instances, or one route assigned twice) + /// is refused as a whole before anything is admitted or removed. + pub async fn reconcile(&self, snapshot: &AssignmentSnapshot) -> Result<(), ControlError> { + if snapshot.instances.len() > MAX_ASSIGNED_INSTANCES { + return Err(ControlError::Invalid(format!( + "snapshot assigns {} instances; at most {MAX_ASSIGNED_INSTANCES} are supported", + snapshot.instances.len() + ))); + } + let mut routes = std::collections::BTreeSet::new(); + for assignment in &snapshot.instances { + if !routes.insert(assignment.route.clone()) { + return Err(ControlError::Invalid( + "snapshot assigns one route more than once".to_string(), + )); + } + } + *self + .desired_routes + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(routes); + let mut rejected = BTreeMap::new(); + for assignment in &snapshot.instances { + let outcome = SandboxInstanceRegistration::from_assignment(assignment) + .map_err(|error| ControlError::Invalid(error.to_string())) + .and_then(|registration| self.register(registration)); + match outcome { + Ok(SandboxInstanceAdmitted::Created) => { + tracing::info!( + route = %assignment.route, + sandbox_id = %assignment.sandbox_id, + "admitted assigned sandbox instance" + ); + } + Ok(SandboxInstanceAdmitted::Unchanged) => {} + Err(error) => { + tracing::warn!( + route = %assignment.route, + sandbox_id = %assignment.sandbox_id, + %error, + "rejected assigned sandbox instance" + ); + rejected.insert( + assignment.route.clone(), + SandboxInstanceStatusReport { + route: assignment.route.clone(), + sandbox_id: assignment.sandbox_id.clone(), + phase: SandboxInstancePhaseReport::Failed { + error: error.to_string(), + }, + ready: false, + }, + ); + } + } + } + *self + .rejected + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = rejected; + self.remove_withdrawn().await; + Ok(()) + } + + /// Remove every hosted instance the latest applied snapshot omits. Also + /// run between snapshots: a removal whose file cleanup failed keeps its + /// key reserved and reports a failure, and the gateway does not resend an + /// unchanged snapshot, so this retries it. Nothing happens before the + /// first snapshot of a process, since nothing is known to be withdrawn. + pub async fn remove_withdrawn(&self) { + let desired = self + .desired_routes + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + let Some(desired) = desired else { + return; + }; + let withdrawn = self + .host + .list() + .into_iter() + .filter(|hosted| !desired.contains(&hosted.route)) + .collect::>(); + futures::future::join_all(withdrawn.iter().map(|hosted| async move { + tracing::info!( + route = %hosted.route, + sandbox_id = %hosted.sandbox_id, + "removing withdrawn sandbox instance" + ); + self.remove(&hosted.route).await; + })) + .await; + } + + /// Whether an instance the latest applied snapshot omits is still hosted, + /// such as one whose removal could not delete its private files. + #[must_use] + pub fn has_withdrawn_leftovers(&self) -> bool { + let desired = self + .desired_routes + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let Some(desired) = desired.as_ref() else { + return false; + }; + self.host + .list() + .iter() + .any(|hosted| !desired.contains(&hosted.route)) + } + + /// Launch material of `registration` with its private files in `dir`. + fn instance_launch( + &self, + registration: SandboxInstanceRegistration, + dir: &Path, + ) -> Result<(crate::SandboxInstanceBindings, SandboxInstanceBody), ControlError> { + let failure = |error: miette::Report| ControlError::Unavailable(error.to_string()); + let main_process = + openshell_core::sandbox_env::MainProcessConfig::decode(®istration.main_process_spec) + .map_err(ControlError::Invalid)?; + let descriptor = registration + .runtime_descriptor + .backend_descriptor() + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let ca_certificate = dir.join("proxy-ca.crt"); + let ca_private_key = dir.join("proxy-ca.key"); + write_private_file( + &ca_certificate, + registration.proxy_ca.certificate_pem.as_bytes(), + ) + .map_err(failure)?; + write_private_file( + &ca_private_key, + registration + .proxy_ca + .private_key_pem + .expose_secret() + .as_bytes(), + ) + .map_err(failure)?; + let trust_dir = dir.join("trust"); + crate::sandbox_instance_host::prepare_private_dir(&trust_dir).map_err(failure)?; + let ssh_socket = dir.join("ssh.sock"); + let config = crate::SandboxInstanceConfig { + command: main_process.command, + workdir: registration.workdir, + timeout_secs: 0, + interactive: main_process.tty, + await_main_process_attachment: main_process.await_main_process_attachment, + sandbox_id: Some(registration.sandbox_id), + sandbox: Some(registration.sandbox_name), + container_image: registration.container_image, + openshell_endpoint: Some(self.gateway_endpoint.clone()), + policy_rules: None, + policy_data: None, + ssh_socket_path: Some(ssh_socket.to_string_lossy().into_owned()), + shared_ssh_socket: false, + // The Kubernetes driver refuses the gateway topology when an + // upstream proxy or provider workload identity is configured, + // and it configures no extra CA for dedicated supervisors. + upstream_proxy_args: + openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs::default(), + additional_ca_bundle: None, + backend_descriptor: descriptor, + auth_bundle: registration.auth, + admitted_isolation_backend: Some(openshell_sandbox_backend::BACKEND_NAME.to_string()), + main_exit_marker: None, + trace_context: registration.trace_context, + // One process cannot hold a per-sandbox workload identity, so + // providers that need one fail closed. + provider_workload_api_socket: None, + proxy_tls: openshell_supervisor_network::run::ProxyTlsFiles { + ca_certificate: Some(ca_certificate), + ca_private_key: Some(ca_private_key), + directory: Some(trust_dir), + }, + }; + crate::sandbox_instance_host::sandbox_instance_launch( + Box::new(crate::backend_setup::RoutedBackendSetup::new( + self.router.clone(), + registration.route, + )), + config, + dir, + ) + .map_err(failure) + } +} + +fn registration_digest( + registration: &SandboxInstanceRegistration, +) -> Result<[u8; 32], ControlError> { + let encoded = serde_json::to_vec(registration) + .map_err(|error| ControlError::Invalid(format!("encode registration: {error}")))?; + openshell_crypto::sha256(&encoded).map_err(|error| ControlError::Unavailable(error.to_string())) +} + +#[cfg(unix)] +fn write_private_file(path: &Path, contents: &[u8]) -> Result<()> { + use miette::{IntoDiagnostic as _, WrapErr as _}; + use std::io::Write as _; + use std::os::unix::fs::OpenOptionsExt as _; + + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(path) + .into_diagnostic() + .wrap_err_with(|| format!("create {}", path.display()))?; + file.write_all(contents).into_diagnostic()?; + file.sync_all().into_diagnostic() +} + +#[cfg(not(unix))] +fn write_private_file(_path: &Path, _contents: &[u8]) -> Result<()> { + Err(miette::miette!("shared supervision requires a Unix host")) +} + +/// How the host reaches the gateway's supervisor host service. +pub struct GatewayLink { + /// Gateway gRPC endpoint; must be `https`, since the supervisor token is + /// a bearer. Server-authenticated TLS follows the sandbox instance + /// conventions (`OPENSHELL_TLS_CA`, `OPENSHELL_GATEWAY_TLS_SERVER_NAME`). + pub endpoint: String, + /// File holding the gateway-minted supervisor token. It is read on every + /// connection attempt, so the gateway's renewals take effect at the next + /// reconnect without a restart. + pub token_file: PathBuf, +} + +impl GatewayLink { + /// Read the current supervisor token. A missing, empty, malformed, or + /// already expired token is an error; the gateway keeps the file fresh, + /// so the caller retries later. + fn read_token(&self) -> Result { + use miette::{IntoDiagnostic as _, WrapErr as _}; + + let contents = std::fs::read_to_string(&self.token_file) + .into_diagnostic() + .wrap_err_with(|| format!("read supervisor token {}", self.token_file.display()))?; + let token = openshell_core::jwt::SecretJwt::parse(contents.trim()) + .map_err(|_| miette::miette!("the supervisor token file holds no token yet"))?; + let expires_at = openshell_core::jwt::parse_exp_secs(token.expose_secret()) + .ok_or_else(|| miette::miette!("the supervisor token is not an expiring JWT"))?; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| { + i64::try_from(elapsed.as_secs()).unwrap_or(i64::MAX) + }); + if expires_at <= now { + return Err(miette::miette!( + "the supervisor token expired {} s ago; waiting for the gateway to renew it", + now.saturating_sub(expires_at) + )); + } + BearerTokenSlot::new(token.expose_secret(), expires_at.saturating_mul(1000)) + .map_err(|error| miette::miette!("{error}")) + } +} + +/// Everything the host reports for one snapshot revision. +fn status_message(host: &SharedHost, applied_revision: u64) -> HostMessage { + HostMessage { + payload: Some(host_message::Payload::Status(HostStatus { + applied_revision, + instances: host + .list() + .iter() + .map(SandboxInstanceStatusReport::to_proto) + .collect(), + })), + } +} + +/// Why one gateway stream ended. +enum StreamEnd { + /// The gateway rejected or closed the stream; reconnect after a backoff. + /// `accepted` records whether the gateway had accepted the host. + Lost { accepted: bool }, +} + +/// Keep one assignment stream open to the gateway until `shutdown` +/// completes, reconnecting with backoff. Hosted instances run on across +/// disconnects. +pub async fn attach_to_gateway( + host: Arc, + link: GatewayLink, + shutdown: impl Future, +) { + let instance_id = uuid::Uuid::new_v4().to_string(); + let mut connection_epoch = 0_u64; + let mut backoff = RECONNECT_BACKOFF_MIN; + tokio::pin!(shutdown); + loop { + connection_epoch += 1; + let hello = HostHello { + instance_id: instance_id.clone(), + connection_epoch, + }; + let StreamEnd::Lost { accepted } = tokio::select! { + end = run_stream(&host, &link, hello) => end, + () = &mut shutdown => return, + }; + if accepted { + backoff = RECONNECT_BACKOFF_MIN; + } + tracing::info!( + retry_in_secs = backoff.as_secs(), + "gateway assignment stream lost; hosted sandbox instances keep running" + ); + tokio::select! { + () = tokio::time::sleep(backoff) => {} + () = &mut shutdown => return, + } + backoff = (backoff * 2).min(RECONNECT_BACKOFF_MAX); + } +} + +/// One connection and stream to the gateway. +/// +/// Snapshots are reconciled on a separate task so heartbeats and status +/// reports keep flowing while instances stop; at most one reconciliation runs +/// at a time, only the newest unapplied snapshot is kept, and the stream does +/// not end until a running reconciliation has finished. +async fn run_stream(host: &Arc, link: &GatewayLink, hello: HostHello) -> StreamEnd { + let lost = |accepted| StreamEnd::Lost { accepted }; + if !link.endpoint.starts_with("https://") { + tracing::error!("the supervisor token requires an https gateway endpoint"); + return lost(false); + } + let token = match link.read_token() { + Ok(token) => token, + Err(error) => { + tracing::warn!(%error, "no usable supervisor token; will retry"); + return lost(false); + } + }; + let connect = openshell_core::grpc_client::connect_gateway_channel(&link.endpoint); + let channel = match tokio::time::timeout(CONNECT_TIMEOUT, connect).await { + Ok(Ok(channel)) => channel, + Ok(Err(error)) => { + tracing::warn!(%error, "could not connect to the gateway"); + return lost(false); + } + Err(_) => { + tracing::warn!("connecting to the gateway timed out"); + return lost(false); + } + }; + let mut client = SupervisorHostClient::with_interceptor(channel, token.interceptor()) + .max_decoding_message_size(MAX_GATEWAY_MESSAGE_BYTES) + .max_encoding_message_size(MAX_HOST_MESSAGE_BYTES); + let (tx, rx) = tokio::sync::mpsc::channel::(OUTBOUND_QUEUE); + let send = |message: HostMessage| { + let tx = tx.clone(); + async move { tx.send_timeout(message, SEND_TIMEOUT).await.is_ok() } + }; + if !send(HostMessage { + payload: Some(host_message::Payload::Hello(hello)), + }) + .await + { + return lost(false); + } + let attach = client.attach(tokio_stream::wrappers::ReceiverStream::new(rx)); + let mut inbound = match tokio::time::timeout(CONNECT_TIMEOUT, attach).await { + Ok(Ok(response)) => response.into_inner(), + Ok(Err(status)) => { + tracing::warn!(%status, "gateway refused the supervisor host stream"); + return lost(false); + } + Err(_) => { + tracing::warn!("gateway did not answer the supervisor host stream in time"); + return lost(false); + } + }; + let mut changes = host.changes(); + changes.mark_unchanged(); + let mut applied_revision = 0_u64; + // Highest snapshot revision received on this stream, applied or not, so a + // snapshot that arrives behind a newer in-flight one is never applied. + let mut received_revision = 0_u64; + let mut accepted = false; + let accept_deadline = tokio::time::Instant::now() + CONNECT_TIMEOUT; + let mut last_heard = tokio::time::Instant::now(); + let mut heartbeat = tokio::time::interval(HEARTBEAT_INTERVAL); + heartbeat.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + // The reconciliation in flight and the newest snapshot waiting for it. + // A retry of withdrawals whose cleanup failed runs through the same slot, + // so at most one reconciliation touches the registry at a time. + let mut applying: Option, ControlError>>> = None; + let mut queued: Option = None; + let start = |snapshot: AssignmentSnapshot| { + let host = host.clone(); + tokio::spawn(async move { + host.reconcile(&snapshot).await?; + Ok(Some(snapshot.revision)) + }) + }; + let retry_withdrawn = || { + let host = host.clone(); + tokio::spawn(async move { + host.remove_withdrawn().await; + Ok(None) + }) + }; + let end = loop { + tokio::select! { + message = inbound.next() => { + let payload = match message { + Some(Ok(message)) => message.payload, + Some(Err(status)) => { + tracing::warn!(%status, "gateway assignment stream failed"); + break lost(accepted); + } + None => break lost(accepted), + }; + last_heard = tokio::time::Instant::now(); + match payload { + Some(host_gateway_message::Payload::Accepted(accepted_message)) => { + accepted = true; + tracing::info!(session = %accepted_message.session_id, "gateway accepted the supervisor host"); + // Report what runs before any snapshot arrives. + if !send(status_message(host, applied_revision)).await { + break lost(accepted); + } + } + Some(host_gateway_message::Payload::Rejected(rejected)) => { + tracing::error!(reason = %rejected.reason, "gateway rejected the supervisor host"); + break lost(accepted); + } + Some(host_gateway_message::Payload::Assignments(snapshot)) => { + if !accepted { + tracing::warn!("gateway sent assignments before accepting the host"); + break lost(accepted); + } + if snapshot.revision <= received_revision { + tracing::debug!(revision = snapshot.revision, received_revision, "ignoring stale assignment snapshot"); + continue; + } + received_revision = snapshot.revision; + if applying.is_some() { + queued = Some(snapshot); + } else { + applying = Some(start(snapshot)); + } + } + Some(host_gateway_message::Payload::Heartbeat(_)) | None => {} + } + } + applied = async { applying.as_mut().expect("polled only while applying").await }, if applying.is_some() => { + applying = None; + match applied { + Ok(Ok(revision)) => { + if let Some(revision) = revision { + applied_revision = revision; + } + changes.mark_unchanged(); + if !send(status_message(host, applied_revision)).await { + break lost(accepted); + } + } + Ok(Err(error)) => { + tracing::warn!(%error, "refused a malformed assignment snapshot"); + } + Err(error) => { + tracing::error!(%error, "assignment reconciliation task failed"); + } + } + if let Some(snapshot) = queued.take() { + applying = Some(start(snapshot)); + } + } + notified = changes.changed() => { + if notified.is_err() { + break lost(accepted); + } + if accepted && !send(status_message(host, applied_revision)).await { + break lost(accepted); + } + } + _ = heartbeat.tick() => { + if !accepted && tokio::time::Instant::now() > accept_deadline { + tracing::warn!("gateway did not accept the supervisor host in time"); + break lost(accepted); + } + if last_heard.elapsed() > GATEWAY_SILENCE_TIMEOUT { + tracing::warn!("gateway assignment stream went silent; reconnecting"); + break lost(accepted); + } + let beat = HostMessage { payload: Some(host_message::Payload::Heartbeat(HostHeartbeat {})) }; + if !send(beat).await { + break lost(accepted); + } + // Withdrawn instances whose cleanup failed are retried at the + // heartbeat cadence, never concurrently with a snapshot. + if applying.is_none() && queued.is_none() && host.has_withdrawn_leftovers() { + applying = Some(retry_withdrawn()); + } + } + } + }; + // Finish the reconciliation this stream started before another stream + // can begin one; the queued snapshot belongs to a session that is gone. + if let Some(applying) = applying { + let _ = applying.await; + } + end +} diff --git a/deploy/helm/openshell/ci/values-supervisor.yaml b/deploy/helm/openshell/ci/values-supervisor.yaml new file mode 100644 index 0000000000..0361016d1b --- /dev/null +++ b/deploy/helm/openshell/ci/values-supervisor.yaml @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# E2E overlay: supervise new sandbox generations with one supervisor process +# shared by the gateway. Layer it on a workspace-mode overlay +# to cover managed or operator namespaces. +gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 2421463d3d..44a1a8a8bb 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -353,6 +353,62 @@ the in-cluster Service DNS, release namespace, service port, and disableTls flag — so the default value works for any release name or namespace without override. */}} +{{/* +Whether the gateway supervisor topology is enabled. Read from the +raw gateway configuration so openshell.effectiveKubernetesConfig can use it. +*/}} +{{- define "openshell.supervisorEnabled" -}} +{{- $kubernetes := get (.Values.gatewayConfig | default dict) "openshell.drivers.kubernetes" | default dict -}} +{{- if eq (get $kubernetes "supervisor_topology" | default "") "gateway" -}}true{{- end -}} +{{- end }} + +{{/* +Name of the shared supervisor Deployment and Service (and, with `-token`, its token Secret): +the fullname with a fixed suffix, within the 63-character Service name limit. +A fullname without room for the suffix keeps a prefix and a hash of the whole +fullname, so the name never collapses into another release resource's name. +*/}} +{{- define "openshell.supervisorName" -}} +{{- $suffix := "-supervisor" -}} +{{- $base := include "openshell.fullname" . -}} +{{- $room := sub 63 (len $suffix) | int -}} +{{- if gt (len $base) $room -}} +{{- $base = printf "%s-%s" ($base | trunc (sub $room 9 | int) | trimSuffix "-") ($base | sha256sum | trunc 8) -}} +{{- end -}} +{{- printf "%s%s" $base $suffix -}} +{{- end }} + +{{/* +Pod labels of the shared supervisor; its Service selects exactly these. +*/}} +{{- define "openshell.supervisorSelectorLabels" -}} +app.kubernetes.io/name: openshell-supervisor +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/component: supervisor +{{- end }} + +{{/* +Namespace selector for the sandbox namespaces of this gateway's workspace +mode: the shared namespace, the managed namespaces it labels, or the +operator-labeled namespaces. +*/}} +{{- define "openshell.supervisorSandboxNamespaces" -}} +{{- $kubernetes := .kubernetes -}} +{{- $mode := get $kubernetes "workspace_mode" | default "shared" -}} +{{- if eq $mode "managed" -}} +matchLabels: + openshell.ai/managed-by: openshell + openshell.ai/gateway-id: {{ get $kubernetes "gateway_id" | quote }} +{{- else if eq $mode "operator" -}} +{{- $label := regexSplit "==?" (get $kubernetes "operator_namespace_label") 2 -}} +matchLabels: + {{ index $label 0 }}: {{ index $label 1 | quote }} +{{- else -}} +matchLabels: + kubernetes.io/metadata.name: {{ get $kubernetes "namespace" | quote }} +{{- end -}} +{{- end }} + {{- define "openshell.grpcEndpoint" -}} {{- if .Values.server.grpcEndpoint -}} {{- .Values.server.grpcEndpoint -}} @@ -561,6 +617,10 @@ like user-supplied schema-v2 fields during compatibility resolution. */}} {{- if not (hasKey $kubernetes "supervisor_image") -}}{{- $_ := set $kubernetes "supervisor_image" (include "openshell.supervisorImage" .) -}}{{- end -}} {{- if not (hasKey $runtimeConfig "boundary_port") -}}{{- $_ := set $runtimeConfig "boundary_port" 5500 -}}{{- end -}} {{- $_ := set $kubernetes "sandbox_runtime" $runtimeConfig -}} +{{/* The chart owns the shared supervisor and the Secret that delivers its token. */}} +{{- if include "openshell.supervisorEnabled" . -}} +{{- $_ := set $kubernetes "shared_supervisor" (dict "service_name" (include "openshell.supervisorName" .) "service_namespace" .Release.Namespace "runtime_port" 5500 "token_secret_name" (printf "%s-token" (include "openshell.supervisorName" .))) -}} +{{- end -}} {{- toYaml $kubernetes -}} {{- end }} @@ -581,6 +641,18 @@ Validate chart values that Helm would otherwise accept silently. {{- if ne (get $gateway "compute_driver" | default "kubernetes") "kubernetes" -}} {{- fail "supervisor_topology requires the kubernetes compute driver" -}} {{- end -}} +{{- if or (get $kubernetes "https_proxy") (get $kubernetes "proxy_ca_bundle") (get $kubernetes "proxy_auth_secret_name") (get $kubernetes "provider_spiffe_workload_api_socket_path") -}} +{{- fail "supervisor_topology=gateway does not support upstream proxy settings or provider SPIFFE identity" -}} +{{- end -}} +{{- if .Values.server.disableTls -}} +{{- fail "supervisor_topology=gateway requires gateway TLS; the supervisor token travels only over TLS" -}} +{{- end -}} +{{- if and (eq (get $kubernetes "workspace_mode" | default "shared") "operator") (not (regexMatch "^[A-Za-z0-9][-A-Za-z0-9_./]*==?[-A-Za-z0-9_.]*$" (get $kubernetes "operator_namespace_label" | default ""))) -}} +{{- fail "supervisor_topology=gateway in operator workspace mode requires operator_namespace_label of the form key=value" -}} +{{- end -}} +{{- end -}} +{{- if hasKey (get (.Values.gatewayConfig | default dict) "openshell.drivers.kubernetes" | default dict) "shared_supervisor" -}} +{{- fail "shared_supervisor is rendered by the chart; set only supervisor_topology" -}} {{- end -}} {{- $workloadKind := include "openshell.workloadKind" . -}} {{- $workload := .Values.workload | default dict -}} diff --git a/deploy/helm/openshell/templates/peer-role.yaml b/deploy/helm/openshell/templates/peer-role.yaml index 3aa2502f46..d7fa148658 100644 --- a/deploy/helm/openshell/templates/peer-role.yaml +++ b/deploy/helm/openshell/templates/peer-role.yaml @@ -18,6 +18,29 @@ rules: - pods verbs: - get + {{- if include "openshell.supervisorEnabled" . }} + # The Kubernetes driver reads the shared supervisor Service to scope each + # sandbox's control egress to the supervisor Pods it selects. + - apiGroups: + - "" + resources: + - services + resourceNames: + - {{ include "openshell.supervisorName" . }} + verbs: + - get + # The gateway delivers the supervisor token it mints by replacing the token + # in the chart-created Secret; it creates no Secrets here. + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - {{ include "openshell.supervisorName" . }}-token + verbs: + - get + - patch + {{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding diff --git a/deploy/helm/openshell/templates/supervisor.yaml b/deploy/helm/openshell/templates/supervisor.yaml index c31fbb610a..d6027d2449 100644 --- a/deploy/helm/openshell/templates/supervisor.yaml +++ b/deploy/helm/openshell/templates/supervisor.yaml @@ -4,34 +4,43 @@ {{- range .Values.imagePullSecrets -}} {{- $imagePullSecrets = append $imagePullSecrets .name -}} {{- end -}} -{{- if eq (get $kubernetes "supervisor_topology" | default "") "gateway" }} +{{- if include "openshell.supervisorEnabled" . }} # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +{{- $shared := get $kubernetes "shared_supervisor" }} +{{- $name := include "openshell.supervisorName" . }} +{{- $endpoint := get $kubernetes "grpc_endpoint" }} +{{- $endpointUrl := urlParse $endpoint }} +{{- $gatewayTls := and (not .Values.server.disableTls) .Values.server.tls.clientTlsSecretName }} +# One supervisor process hosting a sandbox instance per sandbox generation. +# The process dials the gateway over TLS with the supervisor token the gateway +# mints for it and receives its assignments from it; nothing but sandbox +# runtimes and kubelet probes connect to it. The Pod holds no Kubernetes +# identity of its own: each instance acts only with the credentials the +# gateway issued for its generation. apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "openshell.fullname" . }}-supervisor + name: {{ $name }} namespace: {{ .Release.Namespace }} labels: {{- include "openshell.labels" . | nindent 4 }} app.kubernetes.io/component: supervisor spec: + # Instances live in this process; a second replica could not serve them. replicas: 1 strategy: type: Recreate selector: matchLabels: - app.kubernetes.io/name: openshell-supervisor - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/component: supervisor + {{- include "openshell.supervisorSelectorLabels" . | nindent 6 }} template: metadata: labels: - app.kubernetes.io/name: openshell-supervisor - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/component: supervisor + {{- include "openshell.supervisorSelectorLabels" . | nindent 8 }} spec: automountServiceAccountToken: false + enableServiceLinks: false terminationGracePeriodSeconds: 30 {{- with ($imagePullSecrets | uniq) }} imagePullSecrets: @@ -50,11 +59,33 @@ spec: args: - --mode=shared - --gateway-endpoint - - {{ include "openshell.grpcEndpoint" . | quote }} + - {{ $endpoint | quote }} - --address - 0.0.0.0:5501 + - --token-file + - /etc/openshell/supervisor-token/token + - --runtime-listen + - 0.0.0.0:{{ get $shared "runtime_port" }} + - --session-root + - /run/openshell/sessions + env: + - name: OPENSHELL_ENDPOINT + value: {{ $endpoint | quote }} + - name: OPENSHELL_LOG_LEVEL + value: info + {{- if $gatewayTls }} + - name: OPENSHELL_TLS_CA + value: /var/run/secrets/openshell-tls/ca.crt + {{- end }} + {{- if eq (get $endpointUrl "scheme") "https" }} + - name: OPENSHELL_GATEWAY_TLS_SERVER_NAME + value: {{ get $endpointUrl "hostname" | quote }} + {{- end }} ports: - - name: supervisor + - name: runtime + containerPort: {{ get $shared "runtime_port" }} + protocol: TCP + - name: health containerPort: 5501 protocol: TCP securityContext: @@ -64,14 +95,114 @@ spec: drop: [ALL] resources: requests: - cpu: 10m - memory: 32Mi + cpu: 50m + memory: 64Mi limits: - memory: 128Mi + memory: 1Gi + # Process health only: the health listener accepts and closes probe + # connections. Each instance's readiness is reported to the gateway + # over the supervisor's own connection to it. readinessProbe: tcpSocket: - port: supervisor + port: health livenessProbe: tcpSocket: - port: supervisor + port: health + volumeMounts: + - name: run + mountPath: /run/openshell + - name: logs + mountPath: /var/log + - name: supervisor-token + mountPath: /etc/openshell/supervisor-token + readOnly: true + {{- if $gatewayTls }} + - name: gateway-ca + mountPath: /var/run/secrets/openshell-tls + readOnly: true + {{- end }} + volumes: + - name: run + emptyDir: {} + - name: logs + emptyDir: {} + - name: supervisor-token + secret: + secretName: {{ $name }}-token + {{- if $gatewayTls }} + - name: gateway-ca + secret: + secretName: {{ .Values.server.tls.clientTlsSecretName }} + items: + - key: ca.crt + path: ca.crt + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "openshell.labels" . | nindent 4 }} + app.kubernetes.io/component: supervisor +spec: + selector: + {{- include "openshell.supervisorSelectorLabels" . | nindent 4 }} + ports: + - name: runtime + port: {{ get $shared "runtime_port" }} + targetPort: runtime + protocol: TCP +{{- if .Values.networkPolicy.enabled }} +--- +# Only sandbox workloads in this gateway's sandbox namespaces reach the routed +# runtime listener. Each sandbox's own egress allowance limits its workload to +# that port, and the routing preface plus the generation's pinned TLS identity +# decide which instance it reaches. The supervisor dials the gateway itself, +# so no ingress from the gateway is needed. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "openshell.labels" . | nindent 4 }} + app.kubernetes.io/component: supervisor +spec: + podSelector: + matchLabels: + {{- include "openshell.supervisorSelectorLabels" . | nindent 6 }} + policyTypes: + - Ingress + - Egress + ingress: + - from: + - namespaceSelector: + {{- include "openshell.supervisorSandboxNamespaces" (dict "kubernetes" $kubernetes) | nindent 12 }} + podSelector: + matchLabels: + openshell.ai/boundary-role: workload + ports: + - protocol: TCP + port: {{ get $shared "runtime_port" }} + # Instances carry policy-approved sandbox egress, as dedicated supervisors + # do, and the host reaches the gateway. + egress: + - {} +{{- end }} +--- +# Delivery of the gateway-minted supervisor token. The chart creates the +# Secret empty; the gateway replaces its `token` on every renewal, and the +# supervisor re-reads the mounted file on each connection attempt. Helm +# upgrades leave the live token in place. +apiVersion: v1 +kind: Secret +metadata: + name: {{ $name }}-token + namespace: {{ .Release.Namespace }} + labels: + {{- include "openshell.labels" . | nindent 4 }} + app.kubernetes.io/component: supervisor +type: Opaque {{- end }} diff --git a/deploy/helm/openshell/tests/supervisor_test.yaml b/deploy/helm/openshell/tests/supervisor_test.yaml index 2276b3469b..0686a800ec 100644 --- a/deploy/helm/openshell/tests/supervisor_test.yaml +++ b/deploy/helm/openshell/tests/supervisor_test.yaml @@ -13,7 +13,7 @@ tests: asserts: - hasDocuments: count: 0 - - it: deploys one isolated idle process in the gateway namespace + - it: deploys one instance host without Kubernetes identity in the gateway namespace template: templates/supervisor.yaml set: gatewayConfig: @@ -21,18 +21,28 @@ tests: supervisor_topology: gateway supervisor_image: example.com/supervisor:shared asserts: + - hasDocuments: + count: 4 + - equal: + path: kind + value: Deployment + documentIndex: 0 - equal: path: metadata.namespace value: gateway + documentIndex: 0 - equal: path: spec.replicas value: 1 + documentIndex: 0 - equal: path: spec.template.metadata.labels["app.kubernetes.io/name"] value: openshell-supervisor + documentIndex: 0 - equal: path: spec.template.spec.containers[0].image value: example.com/supervisor:shared + documentIndex: 0 - equal: path: spec.template.spec.containers[0].args value: @@ -41,30 +51,204 @@ tests: - https://openshell.gateway.svc.cluster.local:8080 - --address - 0.0.0.0:5501 - - equal: + - --token-file + - /etc/openshell/supervisor-token/token + - --runtime-listen + - 0.0.0.0:5500 + - --session-root + - /run/openshell/sessions + documentIndex: 0 + - lengthEqual: path: spec.template.spec.containers[0].ports - value: - - name: supervisor - containerPort: 5501 - protocol: TCP + count: 2 + documentIndex: 0 - equal: path: spec.template.spec.containers[0].readinessProbe value: tcpSocket: - port: supervisor + port: health + documentIndex: 0 - equal: path: spec.template.spec.containers[0].livenessProbe value: tcpSocket: - port: supervisor - - notExists: + port: health + documentIndex: 0 + - contains: path: spec.template.spec.containers[0].volumeMounts - - notExists: - path: spec.template.spec.volumes + content: + name: run + mountPath: /run/openshell + documentIndex: 0 + - equal: + path: spec.ports + value: + - name: runtime + port: 5500 + targetPort: runtime + protocol: TCP + documentIndex: 1 - equal: path: spec.template.spec.automountServiceAccountToken value: false - - it: renders the same setting into gateway TOML + documentIndex: 0 + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_ENDPOINT + value: https://openshell.gateway.svc.cluster.local:8080 + documentIndex: 0 + - contains: + path: spec.template.spec.containers[0].env + content: + name: OPENSHELL_GATEWAY_TLS_SERVER_NAME + value: openshell.gateway.svc.cluster.local + documentIndex: 0 + - contains: + path: spec.template.spec.volumes + content: + name: supervisor-token + secret: + secretName: openshell-supervisor-token + documentIndex: 0 + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: supervisor-token + mountPath: /etc/openshell/supervisor-token + readOnly: true + documentIndex: 0 + - equal: + path: kind + value: Secret + documentIndex: 3 + - equal: + path: metadata.name + value: openshell-supervisor-token + documentIndex: 3 + - equal: + path: type + value: Opaque + documentIndex: 3 + - notExists: + path: data + documentIndex: 3 + - notExists: + path: stringData + documentIndex: 3 + - contains: + path: spec.template.spec.volumes + content: + name: gateway-ca + secret: + secretName: openshell-client-tls + items: + - key: ca.crt + path: ca.crt + documentIndex: 0 + - equal: + path: kind + value: Service + documentIndex: 1 + - equal: + path: spec.selector + value: + app.kubernetes.io/name: openshell-supervisor + app.kubernetes.io/instance: openshell + app.kubernetes.io/component: supervisor + documentIndex: 1 + - equal: + path: kind + value: NetworkPolicy + documentIndex: 2 + - lengthEqual: + path: spec.ingress + count: 1 + documentIndex: 2 + - equal: + path: spec.ingress[0] + value: + from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: gateway + podSelector: + matchLabels: + openshell.ai/boundary-role: workload + ports: + - protocol: TCP + port: 5500 + documentIndex: 2 + - it: admits runtimes only from this gateway's managed namespaces + template: templates/supervisor.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + workspace_mode: managed + gateway_id: gw-a + asserts: + - equal: + path: spec.ingress[0].from[0].namespaceSelector + value: + matchLabels: + openshell.ai/managed-by: openshell + openshell.ai/gateway-id: gw-a + documentIndex: 2 + - it: admits runtimes only from operator-labeled namespaces + template: templates/supervisor.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + workspace_mode: operator + operator_namespace_label: example.com/openshell-workspace=true + asserts: + - equal: + path: spec.ingress[0].from[0].namespaceSelector + value: + matchLabels: + example.com/openshell-workspace: "true" + documentIndex: 2 + - it: rejects operator namespace selectors it cannot scope + template: templates/supervisor.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + workspace_mode: operator + operator_namespace_file: /etc/openshell/namespaces + asserts: + - failedTemplate: + errorMessage: supervisor_topology=gateway in operator workspace mode requires operator_namespace_label of the form key=value + - it: keeps a distinct name when the release fullname leaves no room for the suffix + template: templates/supervisor.yaml + set: + fullnameOverride: openshell-gateway-release-with-a-very-very-long-name-0057 + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + asserts: + - matchRegex: + path: metadata.name + pattern: ^openshell-gateway-release-with-a-very-very-[0-9a-f]{8}-supervisor$ + documentIndex: 1 + - notEqual: + path: metadata.name + value: openshell-gateway-release-with-a-very-very-long-name-0057 + documentIndex: 1 + - it: requires gateway TLS for the supervisor token + template: templates/supervisor.yaml + set: + server: + disableTls: true + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + asserts: + - failedTemplate: + errorMessage: supervisor_topology=gateway requires gateway TLS; the supervisor token travels only over TLS + - it: renders the chart-owned shared supervisor connection into gateway TOML template: templates/gateway-config.yaml set: gatewayConfig: @@ -74,6 +258,31 @@ tests: - matchRegex: path: data["gateway.toml"] pattern: 'supervisor_topology = "gateway"' + - matchRegex: + path: data["gateway.toml"] + pattern: 'shared_supervisor = \{ runtime_port = 5500, service_name = "openshell-supervisor", service_namespace = "gateway", token_secret_name = "openshell-supervisor-token" \}' + - it: rejects a hand-written shared supervisor connection + template: templates/supervisor.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + shared_supervisor: + service_name: elsewhere + asserts: + - failedTemplate: + errorMessage: shared_supervisor is rendered by the chart; set only supervisor_topology + - it: rejects settings the shared supervisor cannot isolate + template: templates/supervisor.yaml + set: + upstreamProxy: + url: http://proxy.example:3128 + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + asserts: + - failedTemplate: + errorMessage: supervisor_topology=gateway does not support upstream proxy settings or provider SPIFFE identity - it: rejects workspace until implemented template: templates/supervisor.yaml set: diff --git a/deploy/helm/openshell/tests/supervisor_wiring_test.yaml b/deploy/helm/openshell/tests/supervisor_wiring_test.yaml new file mode 100644 index 0000000000..b37c16e295 --- /dev/null +++ b/deploy/helm/openshell/tests/supervisor_wiring_test.yaml @@ -0,0 +1,106 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +suite: supervisor gateway wiring +templates: + - templates/certgen.yaml + - templates/peer-role.yaml + - templates/statefulset.yaml + - templates/gateway-config.yaml +release: + name: openshell + namespace: gateway +tests: + - it: issues no extra PKI for the shared supervisor + template: templates/certgen.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + asserts: + - lengthEqual: + path: spec.template.spec.containers + count: 1 + documentIndex: 3 + - equal: + path: spec.template.spec.containers[0].name + value: certgen + documentIndex: 3 + - it: needs no hook for the shared supervisor when the gateway PKI is external + template: templates/certgen.yaml + set: + pkiInitJob: + enabled: false + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + asserts: + - hasDocuments: + count: 0 + - it: lets the gateway read only the shared supervisor Service + template: templates/peer-role.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + asserts: + - contains: + path: rules + content: + apiGroups: + - "" + resources: + - services + resourceNames: + - openshell-supervisor + verbs: + - get + documentIndex: 0 + - it: lets the gateway replace only the supervisor token in its Secret + template: templates/peer-role.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + asserts: + - contains: + path: rules + content: + apiGroups: + - "" + resources: + - secrets + resourceNames: + - openshell-supervisor-token + verbs: + - get + - patch + documentIndex: 0 + - it: grants no supervisor Secret access without the topology + template: templates/peer-role.yaml + asserts: + - notContains: + path: rules + content: + apiGroups: + - "" + resources: + - secrets + resourceNames: + - openshell-supervisor-token + verbs: + - get + - patch + documentIndex: 0 + - it: keeps the supervisor token Secret off the gateway + template: templates/statefulset.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + supervisor_topology: gateway + asserts: + - notContains: + path: spec.template.spec.volumes + content: + name: supervisor-token + secret: + secretName: openshell-supervisor-token diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index e75aa25ec2..1dc6f0b1cf 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -36,6 +36,8 @@ e2e-kubernetes-ha = ["e2e-kubernetes"] e2e-kubernetes-credential-drivers = ["e2e-kubernetes"] e2e-kubernetes-workspace-managed = ["e2e-kubernetes"] e2e-kubernetes-workspace-operator = ["e2e-kubernetes"] +# One supervisor process shared by the gateway's sandboxes. +e2e-kubernetes-shared-supervisor = ["e2e-kubernetes", "e2e-host-gateway"] e2e-podman = ["e2e", "e2e-host-gateway", "e2e-local-container-driver"] e2e-podman-gpu = ["e2e-podman", "e2e-gpu"] e2e-oidc-pkce = [] diff --git a/e2e/rust/tests/kubernetes_shared_supervisor.rs b/e2e/rust/tests/kubernetes_shared_supervisor.rs new file mode 100644 index 0000000000..4835327ddc --- /dev/null +++ b/e2e/rust/tests/kubernetes_shared_supervisor.rs @@ -0,0 +1,742 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![cfg(feature = "e2e-kubernetes-shared-supervisor")] + +//! Two sandboxes supervised by one shared supervisor process +//! (`supervisor_topology = "gateway"`). +//! +//! Each sandbox keeps its own policy and provider credential, stopping one +//! leaves the other supervised, and the shared runtime listener refuses +//! connections that name a stale or unknown sandbox instance route. + +use std::fmt::Write as _; +use std::io::Write as _; +use std::process::Stdio; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use openshell_e2e::harness::binary::openshell_cmd; +use openshell_e2e::harness::sandbox::SandboxGuard; +use tempfile::NamedTempFile; +use tokio::io::{AsyncBufReadExt as _, AsyncReadExt as _, AsyncWriteExt as _}; +use tokio::net::{TcpListener, TcpStream}; + +const TEST_HOST: &str = "host.openshell.internal"; +const ALPHA: &str = "e2e-shared-alpha"; +const BETA: &str = "e2e-shared-beta"; +const TOKEN_ENV: &str = "E2E_SHARED_SUPERVISOR_TOKEN"; +const TOPOLOGY_ANNOTATION: &str = "openshell.ai/sandbox-runtime-supervisor-topology"; +/// Instance route annotation; the key keeps its established `-session` name. +const INSTANCE_ROUTE_ANNOTATION: &str = "openshell.ai/sandbox-runtime-supervisor-session"; +const SUPERVISOR_COMPONENT: &str = "app.kubernetes.io/component=supervisor"; +const RUNTIME_PORT: u16 = 5500; + +fn release_namespace() -> String { + std::env::var("OPENSHELL_E2E_KUBE_NAMESPACE").unwrap_or_else(|_| "openshell".to_string()) +} + +fn kubectl_command() -> tokio::process::Command { + let mut command = tokio::process::Command::new("kubectl"); + if let Ok(context) = std::env::var("OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE") + && !context.trim().is_empty() + { + command.arg("--context").arg(context); + } + command +} + +async fn kubectl(args: &[&str]) -> String { + let output = kubectl_command() + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await + .unwrap_or_else(|error| panic!("spawn kubectl {args:?}: {error}")); + assert!( + output.status.success(), + "kubectl {args:?} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).into_owned() +} + +async fn cli(args: &[&str]) -> (bool, String) { + let output = openshell_cmd() + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await + .expect("spawn openshell CLI"); + ( + output.status.success(), + format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ), + ) +} + +/// The Sandbox resource of an `OpenShell` sandbox, in whichever namespace the +/// workspace mode placed it. +async fn sandbox_resource(name: &str) -> serde_json::Value { + let output = kubectl(&[ + "get", + "sandboxes.agents.x-k8s.io", + "--all-namespaces", + "-l", + &format!("openshell.ai/sandbox-name={name}"), + "-o", + "json", + ]) + .await; + let list: serde_json::Value = serde_json::from_str(&output).expect("parse Sandbox list"); + let [resource] = list["items"].as_array().expect("Sandbox items").as_slice() else { + panic!("expected one Sandbox resource for {name}: {list}"); + }; + resource.clone() +} + +/// The shared-supervisor sandbox instance route recorded for a sandbox's +/// current generation. +async fn instance_route(name: &str) -> String { + let resource = sandbox_resource(name).await; + let annotations = &resource["metadata"]["annotations"]; + assert_eq!( + annotations[TOPOLOGY_ANNOTATION], "gateway", + "{name} is not supervised by the shared supervisor: {annotations}" + ); + annotations[INSTANCE_ROUTE_ANNOTATION] + .as_str() + .unwrap_or_else(|| panic!("{name} has no sandbox instance route: {annotations}")) + .to_string() +} + +/// UID and restart count of the one shared supervisor Pod. +async fn shared_supervisor_pod() -> (String, i64) { + let output = kubectl(&[ + "-n", + &release_namespace(), + "get", + "pods", + "-l", + SUPERVISOR_COMPONENT, + "-o", + "json", + ]) + .await; + let list: serde_json::Value = serde_json::from_str(&output).expect("parse Pod list"); + let pods = list["items"] + .as_array() + .expect("Pod items") + .iter() + .filter(|pod| pod["metadata"]["deletionTimestamp"].is_null()) + .collect::>(); + let [pod] = pods.as_slice() else { + panic!("expected exactly one shared supervisor Pod: {list}"); + }; + let restarts = pod["status"]["containerStatuses"] + .as_array() + .into_iter() + .flatten() + .map(|status| status["restartCount"].as_i64().unwrap_or_default()) + .sum(); + ( + pod["metadata"]["uid"] + .as_str() + .expect("supervisor Pod UID") + .to_string(), + restarts, + ) +} + +/// No dedicated supervisor Pod serves any sandbox. +async fn assert_no_dedicated_supervisors() { + let output = kubectl(&[ + "get", + "pods", + "--all-namespaces", + "-l", + "openshell.ai/boundary-role=supervisor", + "-o", + "name", + ]) + .await; + assert!( + output.trim().is_empty(), + "dedicated supervisor Pods exist under the gateway topology:\n{output}" + ); +} + +/// Credential a host-side endpoint received, as reported back to a sandbox. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Seen { + Alpha, + Beta, + Placeholder, + Other, +} + +impl Seen { + const fn marker(self) -> &'static str { + match self { + Self::Alpha => "SAW_ALPHA", + Self::Beta => "SAW_BETA", + Self::Placeholder => "SAW_PLACEHOLDER", + Self::Other => "SAW_OTHER", + } + } +} + +/// Host endpoint that reports which provider credential each request carried. +struct CredentialEndpoint { + port: u16, + seen: Arc>>, + task: tokio::task::JoinHandle<()>, +} + +impl CredentialEndpoint { + async fn start(alpha: &str, beta: &str) -> Self { + let listener = TcpListener::bind(("0.0.0.0", 0)) + .await + .expect("bind credential endpoint"); + let port = listener.local_addr().expect("endpoint address").port(); + let seen = Arc::new(Mutex::new(Vec::new())); + let (alpha, beta, record) = (alpha.to_string(), beta.to_string(), seen.clone()); + let task = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + let (alpha, beta, record) = (alpha.clone(), beta.clone(), record.clone()); + tokio::spawn(async move { + let _ = answer(stream, &alpha, &beta, &record).await; + }); + } + }); + Self { port, seen, task } + } + + fn seen(&self) -> Vec { + self.seen.lock().unwrap().clone() + } +} + +impl Drop for CredentialEndpoint { + fn drop(&mut self) { + self.task.abort(); + } +} + +async fn answer( + stream: TcpStream, + alpha: &str, + beta: &str, + record: &Mutex>, +) -> std::io::Result<()> { + let mut reader = tokio::io::BufReader::new(stream); + let mut authorization = String::new(); + loop { + let mut line = String::new(); + if reader.read_line(&mut line).await? == 0 || line == "\r\n" { + break; + } + if let Some((name, value)) = line.split_once(':') + && name.eq_ignore_ascii_case("authorization") + { + authorization = value.trim().to_string(); + } + } + let seen = if authorization.contains(alpha) { + Seen::Alpha + } else if authorization.contains(beta) { + Seen::Beta + } else if authorization.contains("openshell:resolve:env:") { + Seen::Placeholder + } else { + Seen::Other + }; + record.lock().unwrap().push(seen); + let body = seen.marker(); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + reader.get_mut().write_all(response.as_bytes()).await +} + +fn temp_yaml(contents: &str) -> NamedTempFile { + let mut file = tempfile::Builder::new() + .suffix(".yaml") + .tempfile() + .expect("create temp file"); + file.write_all(contents.as_bytes()) + .expect("write temp file"); + file.flush().expect("flush temp file"); + file +} + +/// A profile whose credential binds only to `port` on the test host. An +/// attached provider also contributes that endpoint to the sandbox's policy. +fn profile(id: &str, port: u16) -> NamedTempFile { + temp_yaml(&format!( + r"id: {id} +display_name: E2E Shared Supervisor +category: other +credentials: + - name: token + env_vars: [{TOKEN_ENV}] + required: true + auth_style: bearer + header_name: authorization +endpoints: + - host: {TEST_HOST} + port: {port} + protocol: rest + access: full +binaries: + - path: /usr/bin/python* + - path: /usr/local/bin/python* + - path: /sandbox/.uv/python/*/bin/python* +" + )) +} + +/// A sandbox policy allowing only `ports` on the test host. +fn policy(ports: &[u16]) -> NamedTempFile { + let endpoints = ports.iter().fold(String::new(), |mut endpoints, port| { + let _ = write!( + endpoints, + " - host: {TEST_HOST}\n port: {port}\n protocol: rest\n access: full\n" + ); + endpoints + }); + temp_yaml(&format!( + r"version: 1 +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] + read_write: [/sandbox, /tmp, /dev/null] +landlock: + compatibility: best_effort +process: + run_as_user: sandbox + run_as_group: sandbox +network_policies: + shared_supervisor: + name: shared_supervisor + endpoints: +{endpoints} binaries: + - path: /usr/bin/python* + - path: /usr/local/bin/python* + - path: /sandbox/.uv/python/*/bin/python* +" + )) +} + +/// Call `port` on the test host, with the sandbox's provider placeholder when +/// `credential` is set, and print what the endpoint saw, or `DENIED`. +fn call_script(port: u16, credential: bool) -> String { + let authorization = if credential { + "Authorization: Bearer {token}\\r\\n" + } else { + "" + }; + format!( + r#" +import os +import socket + +try: + with socket.create_connection(({TEST_HOST:?}, {port}), timeout=15) as sock: + token = os.environ.get({TOKEN_ENV:?}, "") + sock.sendall( + (f"GET /credential HTTP/1.1\r\nHost: {TEST_HOST}:{port}\r\n" + f"{authorization}Connection: close\r\n\r\n").encode("ascii") + ) + sock.settimeout(15) + response = b"" + while True: + chunk = sock.recv(4096) + if not chunk: + break + response += chunk + markers = [m for m in ("SAW_ALPHA", "SAW_BETA", "SAW_PLACEHOLDER", "SAW_OTHER") if m.encode() in response] + print(markers[0] if markers else "DENIED") +except OSError: + print("DENIED") +"# + ) +} + +async fn call(sandbox: &SandboxGuard, port: u16) -> String { + call_with(sandbox, port, true).await +} + +async fn call_with(sandbox: &SandboxGuard, port: u16, credential: bool) -> String { + let output = sandbox + .exec(&["python3", "-c", &call_script(port, credential)]) + .await + .unwrap_or_else(|error| panic!("call from {}: {error}", sandbox.name)); + output + .lines() + .map(str::trim) + .find(|line| line.starts_with("SAW_") || *line == "DENIED") + .unwrap_or_else(|| panic!("no call result from {}:\n{output}", sandbox.name)) + .to_string() +} + +async fn delete_until_gone(args: &[&str]) { + for _ in 0..40 { + let (deleted, output) = cli(args).await; + if deleted || output.to_lowercase().contains("not found") { + return; + } + tokio::time::sleep(Duration::from_millis(500)).await; + } + eprintln!("'{}' did not settle", args.join(" ")); +} + +async fn remove_providers() { + for name in [ALPHA, BETA] { + delete_until_gone(&["provider", "delete", name]).await; + delete_until_gone(&["profile", "delete", name]).await; + } +} + +/// Install provider `name`, of a profile with the same name, holding `secret` +/// for `port`. +async fn install_provider(name: &str, port: u16, secret: &str) { + let profile = profile(name, port); + let (imported, output) = cli(&[ + "profile", + "import", + "--file", + profile.path().to_str().expect("UTF-8 profile path"), + ]) + .await; + assert!(imported, "profile import {name} failed:\n{output}"); + let credential = format!("{TOKEN_ENV}={secret}"); + let (created, output) = cli(&[ + "provider", + "create", + "--name", + name, + "--type", + name, + "--credential", + &credential, + ]) + .await; + assert!(created, "provider create {name} failed:\n{output}"); +} + +/// Bound on the shared supervisor answering a routed connection: its own +/// handshake deadline is 5 s. +const PROBE_DEADLINE: Duration = Duration::from_secs(10); + +/// What the shared supervisor did with a connection that named a route. +#[derive(Debug, PartialEq, Eq)] +enum RouteAnswer { + /// Closed or reset the connection before sending anything. + Refused, + /// Started the TLS handshake for the route's pinned identity; the first + /// bytes it sent. + Challenged(Vec), +} + +/// A connection closed or reset by the peer. +fn is_closed(error: &std::io::Error) -> bool { + matches!( + error.kind(), + std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::ConnectionAborted + | std::io::ErrorKind::BrokenPipe + | std::io::ErrorKind::UnexpectedEof + ) +} + +/// Port-forward to the shared supervisor's routed runtime listener. +struct RuntimeListener { + child: tokio::process::Child, + port: u16, + stderr: Arc>>, +} + +impl RuntimeListener { + async fn forward() -> Self { + let service = format!( + "svc/{}-supervisor", + std::env::var("OPENSHELL_E2E_KUBE_RELEASE").unwrap_or_else(|_| "openshell".into()) + ); + let mut child = kubectl_command() + .args([ + "-n", + &release_namespace(), + "port-forward", + &service, + &format!(":{RUNTIME_PORT}"), + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .spawn() + .expect("spawn kubectl port-forward"); + let stderr = Arc::new(Mutex::new(Vec::new())); + let mut errors = tokio::io::BufReader::new(child.stderr.take().expect("stderr")).lines(); + let recorded = stderr.clone(); + tokio::spawn(async move { + while let Ok(Some(line)) = errors.next_line().await { + recorded.lock().unwrap().push(line); + } + }); + let mut lines = tokio::io::BufReader::new(child.stdout.take().expect("stdout")).lines(); + let port = tokio::time::timeout(Duration::from_secs(30), async { + while let Some(line) = lines.next_line().await.expect("read port-forward") { + // "Forwarding from 127.0.0.1:43127 -> 5500" + if let Some(port) = line + .strip_prefix("Forwarding from 127.0.0.1:") + .and_then(|rest| rest.split_whitespace().next()) + .and_then(|port| port.parse().ok()) + { + return port; + } + } + panic!("port-forward ended before it was ready"); + }) + .await + .expect("port-forward ready"); + // kubectl reports every forwarded connection on stdout and exits if + // that pipe closes. + tokio::spawn(async move { while let Ok(Some(_)) = lines.next_line().await {} }); + Self { + child, + port, + stderr, + } + } + + /// Every connection reached the supervisor Pod: kubectl reports a failed + /// pod-side dial on stderr, and the local side then sees the same EOF as a + /// refusal. + async fn assert_forwarded_every_connection(&mut self) { + tokio::time::sleep(Duration::from_millis(500)).await; + let failed = self.stderr.lock().unwrap().iter().any(|line| { + line.contains("an error occurred forwarding") || line.contains("error forwarding port") + }); + assert!( + !failed, + "a probe never reached the supervisor; {}", + self.diagnostics() + ); + } + + /// The forwarder's state, for failure messages. + fn diagnostics(&mut self) -> String { + format!( + "kubectl port-forward status {:?}, stderr {:?}", + self.child.try_wait(), + self.stderr.lock().unwrap() + ) + } + + /// Name `route` in a routing preface and report how the supervisor + /// answered. A challenged connection is then sent `reply` in place of the + /// pinned TLS identity and must be closed. Silence fails the test: a + /// listener that hangs has not rejected anything. + async fn probe(&mut self, route: &str, reply: &[u8]) -> RouteAnswer { + let mut stream = TcpStream::connect(("127.0.0.1", self.port)) + .await + .expect("connect to forwarded runtime listener"); + let mut preface = b"OSRP\x01".to_vec(); + preface.push(u8::try_from(route.len()).expect("route length")); + preface.extend_from_slice(route.as_bytes()); + stream.write_all(&preface).await.expect("send preface"); + let mut answer = vec![0_u8; 5]; + let read = match tokio::time::timeout(PROBE_DEADLINE, stream.read(&mut answer)).await { + Ok(Ok(read)) => read, + Ok(Err(error)) if is_closed(&error) => 0, + Ok(Err(error)) => panic!("read the answer to route {route}: {error}"), + Err(elapsed) => panic!( + "route {route} was neither refused nor challenged ({elapsed}, {PROBE_DEADLINE:?}); {}", + self.diagnostics() + ), + }; + if read == 0 { + return RouteAnswer::Refused; + } + answer.truncate(read); + if let Err(error) = stream.write_all(reply).await { + assert!(is_closed(&error), "answer route {route}: {error}"); + } + let mut rest = Vec::new(); + match tokio::time::timeout(PROBE_DEADLINE, stream.read_to_end(&mut rest)).await { + Ok(Ok(_)) => {} + Ok(Err(error)) if is_closed(&error) => {} + Ok(Err(error)) => panic!("wait for route {route} to close: {error}"), + Err(elapsed) => panic!( + "route {route} stayed open without its pinned TLS identity ({elapsed}, {PROBE_DEADLINE:?}); {}", + self.diagnostics() + ), + } + RouteAnswer::Challenged(answer) + } +} + +impl Drop for RuntimeListener { + fn drop(&mut self) { + let _ = self.child.start_kill(); + } +} + +/// A fatal TLS `handshake_failure` alert: a peer without the pinned identity. +const TLS_HANDSHAKE_FAILURE: &[u8] = b"\x15\x03\x03\x00\x02\x02\x28"; +const TLS_HANDSHAKE_RECORD: u8 = 0x16; + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn one_supervisor_isolates_and_outlives_concurrent_sandbox_instances() { + let nonce = format!("{:08x}", rand::random::()); + let (alpha_secret, beta_secret) = (format!("alpha-{nonce}"), format!("beta-{nonce}")); + let alpha_endpoint = CredentialEndpoint::start(&alpha_secret, &beta_secret).await; + let beta_endpoint = CredentialEndpoint::start(&alpha_secret, &beta_secret).await; + let plain_endpoint = CredentialEndpoint::start(&alpha_secret, &beta_secret).await; + remove_providers().await; + install_provider(ALPHA, alpha_endpoint.port, &alpha_secret).await; + install_provider(BETA, beta_endpoint.port, &beta_secret).await; + let (supervisor_uid, supervisor_restarts) = shared_supervisor_pod().await; + + // Only alpha's own policy allows the plain endpoint. + let alpha_policy = policy(&[alpha_endpoint.port, plain_endpoint.port]); + let beta_policy = policy(&[beta_endpoint.port]); + let keep = ["sh", "-c", "echo shared-ready; exec sleep infinity"]; + let create = |policy: &NamedTempFile, provider: &str| { + let policy = policy + .path() + .to_str() + .expect("UTF-8 policy path") + .to_string(); + let provider = provider.to_string(); + async move { + SandboxGuard::create_keep_with_args( + &["--policy", &policy, "--provider", &provider], + &keep, + "shared-ready", + ) + .await + } + }; + let (alpha, beta) = tokio::join!(create(&alpha_policy, ALPHA), create(&beta_policy, BETA)); + let mut alpha = alpha.expect("create alpha sandbox"); + let mut beta = beta.expect("create beta sandbox"); + + // Both generations are sandbox instances on the one shared supervisor + // process. + let alpha_route = instance_route(&alpha.name).await; + let beta_route = instance_route(&beta.name).await; + assert_ne!(alpha_route, beta_route); + assert_no_dedicated_supervisors().await; + assert_eq!(shared_supervisor_pod().await.0, supervisor_uid); + + // Each instance injects only its own credential and enforces only its own + // policy, including the endpoint its own provider contributes. + assert_eq!(call(&alpha, alpha_endpoint.port).await, "SAW_ALPHA"); + assert_eq!(call(&beta, beta_endpoint.port).await, "SAW_BETA"); + assert_eq!(call(&alpha, beta_endpoint.port).await, "DENIED"); + assert_eq!(call(&beta, alpha_endpoint.port).await, "DENIED"); + assert_eq!( + call_with(&alpha, plain_endpoint.port, false).await, + "SAW_OTHER" + ); + assert_eq!(call_with(&beta, plain_endpoint.port, false).await, "DENIED"); + assert_eq!(alpha_endpoint.seen(), [Seen::Alpha]); + assert_eq!(beta_endpoint.seen(), [Seen::Beta]); + assert_eq!(plain_endpoint.seen(), [Seen::Other]); + + // Stopping alpha removes only its sandbox instance. + let (stopped, output) = cli(&["sandbox", "stop", &alpha.name]).await; + assert!(stopped, "sandbox stop failed:\n{output}"); + assert_eq!(call(&beta, beta_endpoint.port).await, "SAW_BETA"); + + // A peer naming a live route must present that instance's pinned TLS + // identity, and failing to does not displace the instance. Stale and + // unknown routes are refused before any handshake. + let mut listener = RuntimeListener::forward().await; + for _ in 0..3 { + match listener.probe(&beta_route, TLS_HANDSHAKE_FAILURE).await { + RouteAnswer::Challenged(answer) => assert_eq!( + answer.first(), + Some(&TLS_HANDSHAKE_RECORD), + "a live route should start the instance's pinned TLS handshake" + ), + RouteAnswer::Refused => { + panic!("a live route should demand the instance's pinned TLS identity") + } + } + } + assert_eq!( + listener.probe(&alpha_route, &[]).await, + RouteAnswer::Refused, + "stale route reached a sandbox instance" + ); + let unknown = format!("rt-{:032x}", rand::random::()); + assert_eq!( + listener.probe(&unknown, &[]).await, + RouteAnswer::Refused, + "unknown route reached a sandbox instance" + ); + assert!(matches!( + listener.probe(&beta_route, TLS_HANDSHAKE_FAILURE).await, + RouteAnswer::Challenged(_) + )); + listener.assert_forwarded_every_connection().await; + drop(listener); + assert_eq!(call(&beta, beta_endpoint.port).await, "SAW_BETA"); + + // Restarting alpha registers a fresh sandbox instance on the same process. + let (started, output) = cli(&["sandbox", "start", &alpha.name]).await; + assert!(started, "sandbox start failed:\n{output}"); + let restarted_route = instance_route(&alpha.name).await; + assert_ne!(restarted_route, alpha_route); + let restarted_call = tokio::time::timeout(Duration::from_secs(120), async { + loop { + match alpha + .exec(&["python3", "-c", &call_script(alpha_endpoint.port, true)]) + .await + { + Ok(output) if output.contains("SAW_ALPHA") => return, + _ => tokio::time::sleep(Duration::from_secs(2)).await, + } + } + }) + .await; + assert!( + restarted_call.is_ok(), + "restarted alpha never regained supervision" + ); + assert_eq!(call(&beta, beta_endpoint.port).await, "SAW_BETA"); + let mut listener = RuntimeListener::forward().await; + assert_eq!( + listener.probe(&alpha_route, &[]).await, + RouteAnswer::Refused, + "the replaced route reached a sandbox instance" + ); + assert!(matches!( + listener + .probe(&restarted_route, TLS_HANDSHAKE_FAILURE) + .await, + RouteAnswer::Challenged(_) + )); + listener.assert_forwarded_every_connection().await; + drop(listener); + + assert_eq!( + shared_supervisor_pod().await, + (supervisor_uid, supervisor_restarts), + "the shared supervisor must keep running throughout" + ); + assert_no_dedicated_supervisors().await; + alpha.cleanup().await; + beta.cleanup().await; + remove_providers().await; +} diff --git a/proto/README.md b/proto/README.md index 61fc8d6456..f029280f8d 100644 --- a/proto/README.md +++ b/proto/README.md @@ -86,6 +86,20 @@ identifies the service being exposed. Run `mise run pre-commit`, the affected SDK checks, and relevant server tests before submitting the change. +## Supervisor host protocol + +[`supervisor_host.proto`](supervisor_host.proto) is the private stream between +a gateway and the supervisor process it shares among its sandboxes: the +supervisor attaches, the gateway sends complete assignment snapshots, and the +supervisor reports what it hosts. It is a version-locked deployment protocol +served on the gateway listener but authorized only for the gateway-minted +supervisor token (`auth_mode: "supervisor"`). It is not imported by +`openshell.proto`, so SDK generation and +gRPC reflection exclude it; the server's schema inventory test classifies it +explicitly. `SandboxInstanceAssignment.launch_material` carries the existing +JSON sandbox instance registration of `openshell-supervisor-control` and is +marked secret. + ## Isolation backend protocol [`isolation_backend.proto`](isolation_backend.proto) defines backend discovery, diff --git a/proto/supervisor_host.proto b/proto/supervisor_host.proto new file mode 100644 index 0000000000..2747dd5749 --- /dev/null +++ b/proto/supervisor_host.proto @@ -0,0 +1,129 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +syntax = "proto3"; + +package openshell.supervisor.host.v1; + +import "google/protobuf/duration.proto"; +import "options.proto"; + +// Private contract between a gateway and the supervisor process it shares +// among its sandboxes. It is a version-locked deployment protocol, not a +// public API: the gateway, the Kubernetes compute driver, and the shared +// supervisor are released together, and no SDK generates bindings for it. +// +// The supervisor host opens one Attach stream, identifies itself, and keeps +// the stream for its lifetime. The gateway owns the desired set of sandbox +// instances and sends it as complete snapshots; the host reconciles its +// registry to each snapshot and reports what it observes. Disconnecting +// never stops hosted instances; only a later snapshot that omits them does. +service SupervisorHost { + // Supervisor-initiated assignment stream. Only the gateway-minted + // supervisor token may call it. + rpc Attach(stream HostMessage) returns (stream HostGatewayMessage) { + option (openshell.options.v1.authorization) = { + auth_mode: "supervisor" + }; + } +} + +// Envelope for host-to-gateway messages. +message HostMessage { + oneof payload { + HostHello hello = 1; + HostStatus status = 2; + HostHeartbeat heartbeat = 3; + } +} + +// Envelope for gateway-to-host messages. +message HostGatewayMessage { + oneof payload { + HostAccepted accepted = 1; + HostRejected rejected = 2; + AssignmentSnapshot assignments = 3; + HostGatewayHeartbeat heartbeat = 4; + } +} + +// First message of every stream. +message HostHello { + // Identifies one host process (for example a boot id). A reconnect from the + // same process keeps it; a restarted process has a new one. + string instance_id = 1; + // Counter the host increments for every connection of one instance_id, so + // the gateway can order reconnects. + uint64 connection_epoch = 2; +} + +// The gateway accepted the host and will send assignment snapshots. +message HostAccepted { + // Scopes every snapshot revision and status report of this stream. + string session_id = 1; + google.protobuf.Duration heartbeat_interval = 2; +} + +// The gateway refuses the host; the stream ends after this message. +message HostRejected { + string reason = 1; +} + +// Complete desired set of sandbox instances. Each snapshot replaces the +// previous one; an instance absent from a snapshot must stop. +message AssignmentSnapshot { + // Monotonic within the session, starting at 1. + uint64 revision = 1; + repeated SandboxInstanceAssignment instances = 2; +} + +// One running generation of a stable Sandbox resource, assigned to the host. +message SandboxInstanceAssignment { + // Route handle the sandbox runtime names; the instance's key on the host. + string route = 1; + string sandbox_id = 2; + // Runtime generation this assignment launches. Never reused. + string generation = 3; + // Trusted launch material for the generation: the JSON-encoded sandbox + // instance registration (runtime descriptor, gateway-issued credentials, + // proxy CA, main process). Never logged or forwarded. + bytes launch_material = 4 [(openshell.options.v1.secret) = true]; +} + +// Everything the host observes, sent after reconciling each snapshot and +// whenever an instance changes. +message HostStatus { + // Highest snapshot revision of this session the host has reconciled to; + // 0 before the first one. + uint64 applied_revision = 1; + // Every hosted instance, including ones that ended but are still being + // removed. + repeated SandboxInstanceStatus instances = 2; +} + +// Lifecycle phase of a hosted sandbox instance. +enum SandboxInstancePhase { + SANDBOX_INSTANCE_PHASE_UNSPECIFIED = 0; + SANDBOX_INSTANCE_PHASE_STARTING = 1; + SANDBOX_INSTANCE_PHASE_SUPERVISING = 2; + SANDBOX_INSTANCE_PHASE_STOPPING = 3; + SANDBOX_INSTANCE_PHASE_EXITED = 4; + SANDBOX_INSTANCE_PHASE_FAILED = 5; +} + +// Observed state of one hosted sandbox instance. +message SandboxInstanceStatus { + string route = 1; + string sandbox_id = 2; + SandboxInstancePhase phase = 3; + // The gateway currently accepts this instance's supervision session. + bool ready = 4; + // Workload exit code; meaningful only for EXITED. + int32 exit_code = 5; + // Sanitized failure; meaningful only for FAILED. + string error = 6; +} + +message HostHeartbeat {} + +message HostGatewayHeartbeat {} diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 164191ff14..88bc93f4a0 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1280,6 +1280,7 @@ credential failures. | Gateway or `-certgen` hook pod rejected with `violates PodSecurity "restricted:latest"` | The rendered container lacks `runAsNonRoot` or `seccompProfile`: an older chart version does not set them, or an override in `securityContext` / `pkiInitJob.securityContext` unset them | `kubectl -n openshell get events --field-selector reason=FailedCreate`; upgrade the chart, then remove overrides that unset those fields | | OpenShift gateway pod fails to start with an SCC/`runAsUser` error (e.g. `unable to validate against any security context constraint`) | Chart's default `podSecurityContext`/`securityContext` hardcodes `runAsUser`/`fsGroup`, which the restricted-v2 SCC rejects; it must instead inject the namespace-assigned UID/GID range | `oc -n openshell describe pod `; deploy with `podSecurityContext: null` and clear `securityContext.runAsUser` (see `deploy/helm/openshell/ci/values-openshift-scc.yaml`) | | OpenShift `-supervisor` Deployment never creates pods after setting `supervisor_topology = "gateway"` (SCC `runAsUser`/`fsGroup` not in the namespace range) | `supervisor.podSecurityContext` defaults `runAsUser`, `runAsGroup`, and `fsGroup` to 65534, outside the namespace's assigned UID/GID ranges | `kubectl -n openshell get events --field-selector reason=FailedCreate`; compare with the `openshift.io/sa.scc.*` annotations from `oc describe ns openshell`; set `supervisor.podSecurityContext.runAsUser`, `.runAsGroup`, and `.fsGroup` to `null`, as `deploy/helm/openshell/ci/values-openshift-scc.yaml` does, and keep `runAsNonRoot` and `seccompProfile` | +| Shared sandboxes stay `Starting` or report `unavailable` readiness with `supervisor_topology = "gateway"`, gateway logs say no shared supervisor host is connected | The `-supervisor` Pod cannot reach or authenticate to the gateway: wrong `OPENSHELL_TLS_CA`, the `-supervisor-token` Secret is missing or still empty (gateway logs `wrote the supervisor token Secret` once it has delivered a token; `no usable supervisor token` in the supervisor means it has not), or the gateway was deployed without TLS or without the sandbox-JWT signing key | `kubectl -n openshell logs deployment/-supervisor` for `gateway accepted the supervisor host` or a refusal; gateway logs for `supervisor host attached`, `wrote the supervisor token Secret`, or `failed to write the supervisor token Secret`; `kubectl -n openshell get secret -supervisor-token -o jsonpath='{.metadata.annotations}'` shows the token expiry; the gateway refuses to start the topology without TLS or `gateway_jwt` | | OpenShift sandbox pod fails to start (`unable to validate against any security context constraint`) | An explicit sandbox UID/GID override or custom security context conflicts with the namespace's SCC range | Inspect pod events and the namespace's `openshift.io/sa.scc.uid-range` and `openshift.io/sa.scc.supplemental-groups` annotations; remove incompatible `sandbox_uid` / `sandbox_gid` overrides so the driver uses the namespace-assigned identity. Current sandbox pods use `restricted-v2` without a privileged SCC grant; see the [OpenShift guide](https://docs.nvidia.com/openshell/latest/kubernetes/openshift) and [RFC 0012](https://github.com/NVIDIA/OpenShell/blob/main/rfc/0012-isolation-backend/README.md#motivation). | | OpenShift self-hosted Vault/OpenBao credential store pod never schedules (waits time out with `no matching resources found`) | The store's Helm chart pins `runAsUser`/`fsGroup`/seccomp, which restricted-v2 rejects, so the StatefulSet controller never creates the pod | Deploy the store's chart in its OpenShift mode (`--set global.openshift=true` for the OpenBao/Vault chart) so the namespace SCC assigns a compliant security context — no manual SCC grant needed | | Vault credential driver returns HTTP 403 / `Vault Kubernetes auth denied the configured role` on provider create | Vault's `auth/kubernetes` method or the gateway login role is not provisioned, or the role is not bound to the gateway service account and namespace | In Vault: `bao auth enable kubernetes` and `bao write auth/kubernetes/config kubernetes_host=... kubernetes_ca_cert=@...`; ensure the login role's `bound_service_account_names`/`bound_service_account_namespaces` match the gateway SA and namespace and its policy grants the credential paths | diff --git a/tasks/test.toml b/tasks/test.toml index dc4fd88fc5..a5a2a03928 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -267,6 +267,12 @@ description = "Run Kubernetes e2e with managed workspace mode (auto-created per- env = { OPENSHELL_E2E_CREDENTIAL_DRIVERS = "1", OPENSHELL_E2E_CREDENTIAL_DRIVER = "kubernetes-secrets", OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-managed.yaml", OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET = "e2e-regcred", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_managed", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-managed" } run = "e2e/rust/e2e-kubernetes.sh" +["e2e:kubernetes:shared-supervisor"] +description = "Run Kubernetes e2e with one supervisor shared by the gateway's sandboxes (gateway supervisor topology)" +env = { OPENSHELL_E2E_KUBE_TEST = "kubernetes_shared_supervisor", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-host-gateway,e2e-kubernetes,e2e-kubernetes-shared-supervisor" } +# Prepend the supervisor overlay to any caller overlays, such as a workspace mode. +run = 'OPENSHELL_E2E_KUBE_EXTRA_VALUES="deploy/helm/openshell/ci/values-supervisor.yaml${OPENSHELL_E2E_KUBE_EXTRA_VALUES:+:$OPENSHELL_E2E_KUBE_EXTRA_VALUES}" e2e/rust/e2e-kubernetes.sh' + ["e2e:kubernetes:workspace-operator"] description = "Run Kubernetes e2e with operator workspace mode (pre-provisioned per-workspace namespaces)" env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-operator.yaml", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_operator", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-operator" }