diff --git a/Cargo.lock b/Cargo.lock index 94cc7b3df6..3b35e45ea8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5339,6 +5339,7 @@ dependencies = [ "openshell-sandbox-backend", "openshell-supervisor-middleware", "openshell-supervisor-middleware-builtins", + "opentelemetry", "opentelemetry-proto", "opentelemetry_sdk", "prost-types", diff --git a/crates/openshell-ocsf/src/events/mod.rs b/crates/openshell-ocsf/src/events/mod.rs index 66038cd9cb..22ecbb6dfc 100644 --- a/crates/openshell-ocsf/src/events/mod.rs +++ b/crates/openshell-ocsf/src/events/mod.rs @@ -169,6 +169,11 @@ impl OcsfEvent { } } + /// Associate this event with other events produced for the same operation. + pub fn set_correlation_uid(&mut self, uid: impl Into) { + self.base_mut().metadata.correlation_uid = Some(uid.into()); + } + /// Returns mutable base data common to all event classes. pub(crate) fn base_mut(&mut self) -> &mut BaseEventData { match self { @@ -281,6 +286,29 @@ mod tests { assert_eq!(deserialized.class_uid(), 2004); } + #[test] + fn correlation_uid_can_be_set_on_any_event_class() { + let ctx = test_sandbox_context(); + let mut event = AuthenticationBuilder::new( + &ctx, + User::other("user-1", "User"), + Service::new("gateway"), + ) + .status(StatusId::Success) + .build(); + + event.set_correlation_uid("request-123"); + + assert_eq!( + event.base().metadata.correlation_uid.as_deref(), + Some("request-123") + ); + assert_eq!( + serde_json::to_value(&event).unwrap()["metadata"]["correlation_uid"], + "request-123" + ); + } + #[test] fn test_roundtrip_application_lifecycle() { let ctx = test_sandbox_context(); diff --git a/crates/openshell-otel/src/lib.rs b/crates/openshell-otel/src/lib.rs index 1f9de6bf17..340936906d 100644 --- a/crates/openshell-otel/src/lib.rs +++ b/crates/openshell-otel/src/lib.rs @@ -34,7 +34,8 @@ pub use ocsf::{OcsfOtlp, OcsfOtlpLayer}; pub use propagation::{ HeaderMapExtractor, MetadataMapInjector, TRACEPARENT_ENV, TRACESTATE_ENV, TraceContextInterceptor, current_trace_context_carrier, current_trace_context_environment, - set_parent_from_environment, set_parent_from_trace_context, trace_context_environment, + set_parent_from_environment, set_parent_from_headers, set_parent_from_trace_context, + trace_context_environment, }; pub use tls::{MAX_TLS_FILE_BYTES, OtlpTls, TlsError, TlsFileError}; diff --git a/crates/openshell-otel/src/ocsf.rs b/crates/openshell-otel/src/ocsf.rs index 63489fde1d..67f8891ead 100644 --- a/crates/openshell-otel/src/ocsf.rs +++ b/crates/openshell-otel/src/ocsf.rs @@ -34,6 +34,7 @@ const MAX_EXPORT_ATTEMPTS: usize = 4; const INITIAL_BACKOFF: Duration = Duration::from_millis(250); const MAX_BACKOFF: Duration = Duration::from_secs(2); const GAP_EVENT: &str = "openshell.ocsf.gap"; +const CORRELATION_UID_ATTRIBUTE: &str = "ocsf.metadata.correlation_uid"; /// Exports OCSF events to an OTLP collector as log records. pub struct OcsfOtlp { @@ -391,6 +392,9 @@ fn log_record( if let Some(uid) = &base.metadata.uid { record.add_attribute("ocsf.metadata.uid", uid.clone()); } + if let Some(uid) = &base.metadata.correlation_uid { + record.add_attribute(CORRELATION_UID_ATTRIBUTE, uid.clone()); + } if let Some(sandbox_id) = event.sandbox_id() { record.add_attribute("openshell.sandbox.id", sandbox_id.to_string()); } @@ -629,6 +633,16 @@ mod tests { assert_eq!(native(&event)["resources"][0]["uid"], "requested-sandbox"); } + #[test] + fn correlation_uid_is_a_stable_otlp_attribute() { + let mut event = event("sandbox-1", SeverityId::Informational); + event.set_correlation_uid("request-123"); + assert_eq!( + attribute(&record(&event), CORRELATION_UID_ATTRIBUTE), + Some(AnyValue::String("request-123".into())) + ); + } + #[test] fn events_without_a_sandbox_carry_no_sandbox_attribute() { let record = record(&event("", SeverityId::Informational)); @@ -785,7 +799,11 @@ mod tests { .build(); let subscriber = tracing_subscriber::registry() .with(crate::layer(&tracer, "ocsf-test")) - .with(otlp.layer()); + .with( + Some(otlp.layer()).with_filter(tracing_subscriber::filter::filter_fn( + |metadata| metadata.target() == OCSF_TARGET, + )), + ); let span_context = tracing::subscriber::with_default(subscriber, || { let span = tracing::info_span!("decision"); span.in_scope(|| ocsf_emit!(event("sandbox-1", SeverityId::Medium))); diff --git a/crates/openshell-otel/src/propagation.rs b/crates/openshell-otel/src/propagation.rs index 02f2bdb120..9553398c9a 100644 --- a/crates/openshell-otel/src/propagation.rs +++ b/crates/openshell-otel/src/propagation.rs @@ -138,6 +138,42 @@ pub fn set_parent_from_trace_context( set_parent_from_carrier(span, &carrier); } +/// Parent `span` under valid W3C trace context from HTTP headers. +/// +/// Returns `false` when `traceparent` is absent, duplicated, invalid, or +/// unsampled. Remote context is correlation data only and does not authenticate +/// its sender. Unsampled remote context is linked rather than adopted so an +/// untrusted caller cannot suppress locally exported enforcement telemetry. +/// `tracestate` is not accepted across this trust boundary. +#[must_use] +pub fn set_parent_from_headers(span: &tracing::Span, headers: &HeaderMap) -> bool { + use opentelemetry::trace::TraceContextExt as _; + + if headers.get_all("traceparent").iter().count() != 1 { + return false; + } + + let Some(traceparent) = headers + .get("traceparent") + .and_then(|value| value.to_str().ok()) + else { + return false; + }; + let carrier = BTreeMap::from([("traceparent".to_string(), traceparent.to_string())]); + let parent = TraceContextPropagator::new().extract(&TraceContextMapExtractor(&carrier)); + let parent_span = parent.span(); + let parent_context = parent_span.span_context(); + if !parent_context.is_valid() { + return false; + } + if !parent_context.is_sampled() { + span.add_link(parent_context.clone()); + return false; + } + + span.set_parent(parent).is_ok() +} + fn set_parent_from_carrier(span: &tracing::Span, carrier: &BTreeMap) { use opentelemetry::trace::TraceContextExt as _; @@ -283,6 +319,116 @@ mod tests { assert_eq!(orphan.parent_span_id, opentelemetry::trace::SpanId::INVALID); } + #[test] + fn http_headers_parent_a_span_only_with_one_valid_traceparent() { + use opentelemetry::trace::TraceContextExt as _; + + let _tracing_lock = crate::test_lock(); + let (_provider, exporter, subscriber) = env_test_exporter(); + let trace_id = "4bf92f3577b34da6a3ce929d0e0e4736"; + let parent_span_id = "00f067aa0ba902b7"; + + tracing::subscriber::with_default(subscriber, || { + let mut headers = HeaderMap::new(); + headers.insert( + "traceparent", + format!("00-{trace_id}-{parent_span_id}-01") + .parse() + .unwrap(), + ); + headers.insert("tracestate", "opaque-vendor=untrusted".parse().unwrap()); + let child = tracing::info_span!("http-child"); + assert!(set_parent_from_headers(&child, &headers)); + let context = child.context(); + assert_eq!( + context.span().span_context().trace_id().to_string(), + trace_id + ); + assert_eq!(context.span().span_context().trace_state().header(), ""); + }); + + let child = exporter + .get_finished_spans() + .unwrap() + .into_iter() + .find(|span| span.name == "http-child") + .unwrap(); + assert_eq!(child.parent_span_id.to_string(), parent_span_id); + } + + #[test] + fn duplicate_and_malformed_http_traceparent_are_rejected() { + let _tracing_lock = crate::test_lock(); + let (_provider, _exporter, subscriber) = env_test_exporter(); + + tracing::subscriber::with_default(subscriber, || { + let mut duplicate = HeaderMap::new(); + duplicate.append( + "traceparent", + "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01" + .parse() + .unwrap(), + ); + duplicate.append( + "traceparent", + "00-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbbbb-01" + .parse() + .unwrap(), + ); + assert!(!set_parent_from_headers( + &tracing::info_span!("duplicate"), + &duplicate + )); + + let mut malformed = HeaderMap::new(); + malformed.insert("traceparent", "invalid".parse().unwrap()); + assert!(!set_parent_from_headers( + &tracing::info_span!("malformed"), + &malformed + )); + }); + } + + #[test] + fn unsampled_http_traceparent_cannot_suppress_local_span_export() { + use opentelemetry::trace::TraceContextExt as _; + + let _tracing_lock = crate::test_lock(); + let (_provider, exporter, subscriber) = env_test_exporter(); + let incoming_trace_id = "4bf92f3577b34da6a3ce929d0e0e4736"; + + tracing::subscriber::with_default(subscriber, || { + let mut unsampled = HeaderMap::new(); + unsampled.insert( + "traceparent", + format!("00-{incoming_trace_id}-00f067aa0ba902b7-00") + .parse() + .unwrap(), + ); + let child = tracing::info_span!("unsampled-child"); + assert!(!set_parent_from_headers(&child, &unsampled)); + let context = child.context(); + let span = context.span(); + let span_context = span.span_context(); + assert!(span_context.is_valid()); + assert!(span_context.is_sampled()); + assert_ne!(span_context.trace_id().to_string(), incoming_trace_id); + }); + + let child = exporter + .get_finished_spans() + .unwrap() + .into_iter() + .find(|span| span.name == "unsampled-child") + .expect("the locally sampled enforcement span is exported"); + assert_eq!(child.parent_span_id, opentelemetry::trace::SpanId::INVALID); + assert_eq!(child.links.len(), 1); + assert_eq!( + child.links[0].span_context.trace_id().to_string(), + incoming_trace_id + ); + } + #[test] fn interceptor_adds_traceparent_only_inside_an_exported_span() { use tonic::service::Interceptor as _; diff --git a/crates/openshell-supervisor-middleware/src/headers.rs b/crates/openshell-supervisor-middleware/src/headers.rs index 056ad32cf7..49d994d790 100644 --- a/crates/openshell-supervisor-middleware/src/headers.rs +++ b/crates/openshell-supervisor-middleware/src/headers.rs @@ -286,25 +286,31 @@ fn is_safe_value(value: &str) -> bool { } fn is_request_protected(name: &str) -> bool { - matches!( - name, - "authorization" - | "proxy-authorization" - | "proxy-authenticate" - | "cookie" - | "host" - | "content-length" - | "transfer-encoding" - | "connection" - | "proxy-connection" - | "keep-alive" - | "te" - | "trailer" - | "upgrade" - ) || name.starts_with("x-amz-") + is_platform_correlation_header(name) + || matches!( + name, + "authorization" + | "proxy-authorization" + | "proxy-authenticate" + | "cookie" + | "host" + | "content-length" + | "transfer-encoding" + | "connection" + | "proxy-connection" + | "keep-alive" + | "te" + | "trailer" + | "upgrade" + ) + || name.starts_with("x-amz-") || name.starts_with("x-openshell-credential") } +fn is_platform_correlation_header(name: &str) -> bool { + name == "x-openshell-request-id" +} + /// Return whether a response header can carry authentication material and must /// never be exposed to middleware. #[must_use] @@ -322,7 +328,8 @@ pub fn is_response_credential_header(name: &str) -> bool { } fn is_response_protected(name: &str) -> bool { - is_response_credential_header(name) + is_platform_correlation_header(name) + || is_response_credential_header(name) || matches!( name, "connection" @@ -408,6 +415,39 @@ mod tests { ); } + #[test] + fn platform_request_id_is_immutable_for_all_http_phases() { + for authority in [ + HeaderAuthority::Request, + HeaderAuthority::Response, + HeaderAuthority::ResponseTrailers, + ] { + let name = "X-OpenShell-Request-ID"; + let write_error = apply( + authority, + &[], + &[], + &[write(name, "forged", ExistingHeaderAction::Overwrite)], + ) + .expect_err("platform request ID write"); + assert_eq!( + write_error, + HeaderMutationError::Protected { + name: name.to_string() + } + ); + + let remove_error = apply(authority, &[], &[], &[remove(name)]) + .expect_err("platform request ID removal"); + assert_eq!( + remove_error, + HeaderMutationError::Protected { + name: name.to_string() + } + ); + } + } + #[test] fn unsafe_header_value_is_rejected() { let error = apply( diff --git a/crates/openshell-supervisor-network/Cargo.toml b/crates/openshell-supervisor-network/Cargo.toml index 7043f30d24..5c9d6ab9a9 100644 --- a/crates/openshell-supervisor-network/Cargo.toml +++ b/crates/openshell-supervisor-network/Cargo.toml @@ -18,6 +18,7 @@ openshell-crypto = { workspace = true } openshell-core = { path = "../openshell-core", features = ["oauth"] } openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-ocsf = { path = "../openshell-ocsf" } +openshell-otel = { path = "../openshell-otel" } openshell-policy = { path = "../openshell-policy" } openshell-policy-schema = { path = "../openshell-policy-schema" } openshell-supervisor-middleware = { path = "../openshell-supervisor-middleware" } @@ -70,9 +71,9 @@ openshell-driver-vm = { path = "../openshell-driver-vm", default-features = fals openshell-otel-test-support = { path = "../openshell-otel-test-support" } openshell-ocsf = { path = "../openshell-ocsf", features = ["test-support"] } openshell-ocsf-validate = { path = "../openshell-ocsf-validate" } -openshell-otel = { path = "../openshell-otel" } openshell-supervisor-middleware-builtins = { path = "../openshell-supervisor-middleware-builtins" } -opentelemetry-proto = { workspace = true } +opentelemetry-proto = { workspace = true, features = ["gen-tonic", "trace"] } +opentelemetry = { workspace = true } opentelemetry_sdk = { workspace = true, features = ["testing"] } temp-env = "0.3" tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/openshell-supervisor-network/README.md b/crates/openshell-supervisor-network/README.md index 38a7fb6add..cd39c07c0a 100644 --- a/crates/openshell-supervisor-network/README.md +++ b/crates/openshell-supervisor-network/README.md @@ -51,3 +51,35 @@ domain event and a detection finding without credential values or placeholders. Explicit body and WebSocket rewrite flags authorize those locations for ordinary static credentials. SigV4 signing retains the endpoint-scoped resolver's existing trusted access. + +## HTTP enforcement correlation + +Inspected HTTP requests create an `openshell.http.request` span with a +`openshell.policy.evaluate` child when policy evaluation runs. A single +valid, sampled W3C `traceparent` becomes its parent. OpenShell treats incoming +trace context as an untrusted correlation hint: malformed or duplicate values +are ignored, `tracestate` is not adopted into the enforcement span, and an +unsampled context becomes a link so a caller cannot suppress local enforcement +telemetry. + +The network supervisor assigns each inspected request a UUID. It stores that +value in the enforcement span, in correlated OCSF events, and in the +`x-openshell-request-id` response header. OpenShell removes workload, +middleware, and upstream attempts to set that header on inspected HTTP paths. +The header is a correlation handle, not proof of an enforcement decision. A +consumer must match it to trusted OpenShell OTLP or OCSF output for the same +sandbox and trace, and must never authorize from the header alone. L4 and +passthrough traffic do not provide this anti-spoofing guarantee. Span +attributes distinguish the policy decision from the enforcement action and +final outcome, such as a local denial, an allowed response, or an upstream or +platform failure. An unfinished request is reported as incomplete. The +attributes never include credentials, query values, or request and response +bodies. + +Each inspected request emits one primary HTTP Activity record when the final +response head is written, after response middleware selects what can be returned. +Informational responses do not commit that record. If the request ends before +the final head, a guard emits it with the available outcome instead. The byte +observer does not buffer headers or alter transport behavior. A receipt confirms +only observation of a response head, not completion of its body. Process crashes +and requests that remain hung cannot be finalized by the guard. diff --git a/crates/openshell-supervisor-network/src/l7/correlation.rs b/crates/openshell-supervisor-network/src/l7/correlation.rs new file mode 100644 index 0000000000..5df3fe338c --- /dev/null +++ b/crates/openshell-supervisor-network/src/l7/correlation.rs @@ -0,0 +1,629 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Per-request correlation for HTTP enforcement events. + +use std::sync::Mutex; +use std::{ + pin::Pin, + task::{Context, Poll}, +}; + +use http::{HeaderMap, HeaderValue}; +use openshell_ocsf::{ + ActionId, Endpoint, HttpActivityBuilder, HttpRequest, HttpResponse, OcsfEvent, SeverityId, + StatusId, +}; + +use crate::l7::provider::L7Request; +use crate::l7::relay::L7EvalContext; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; + +#[derive(Debug)] +pub struct RequestCorrelation { + request_id: String, + span: tracing::Span, + // Keep the audit payload off the async state machine. + state: Box>, +} + +#[derive(Debug)] +struct RequestState { + primary: Option, + status: Option, + upstream_status: Option, + outcome: Option<&'static str>, + policy_span: Option, +} + +impl RequestCorrelation { + #[must_use] + pub(crate) fn new(request: &L7Request, ctx: &L7EvalContext) -> Self { + Self::build(request, ctx, uuid::Uuid::new_v4().to_string()) + } + + #[cfg(test)] + #[must_use] + pub(crate) fn with_request_id( + request: &L7Request, + ctx: &L7EvalContext, + request_id: String, + ) -> Self { + Self::build(request, ctx, request_id) + } + + fn build(request: &L7Request, ctx: &L7EvalContext, request_id: String) -> Self { + let method = bounded_http_method(&request.action); + let span = tracing::info_span!( + "openshell.http.request", + otel.kind = "server", + otel.status_code = tracing::field::Empty, + "openshell.request.id" = %request_id, + "http.request.method" = method, + "server.address" = %ctx.host, + "server.port" = ctx.port, + "openshell.policy.name" = %ctx.policy_name, + "openshell.target.reached" = false, + "openshell.policy.decision" = tracing::field::Empty, + "openshell.enforcement.action" = tracing::field::Empty, + "openshell.http.outcome" = tracing::field::Empty, + "http.response.status_code" = tracing::field::Empty, + ); + let headers = trace_headers(&request.raw_header); + let _ = openshell_otel::set_parent_from_headers(&span, &headers); + let fallback = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .http_request(HttpRequest::new(method, ctx.ocsf_url("/"))) + .dst_endpoint(Endpoint::from_domain(&ctx.host, ctx.port)) + .firewall_rule(&ctx.policy_name, "l7") + .message("HTTP request ended before a response was selected") + .build(); + Self { + request_id, + span, + state: Box::new(Mutex::new(RequestState { + primary: Some(fallback), + status: None, + upstream_status: None, + outcome: None, + policy_span: None, + })), + } + } + + #[must_use] + pub(crate) fn request_id(&self) -> &str { + &self.request_id + } + + pub(crate) fn observe_response<'a, C>(&'a self, client: &'a mut C) -> AuditedClient<'a, C> { + AuditedClient { + inner: client, + correlation: self, + head: ResponseHead::default(), + } + } + + pub(crate) fn record_policy_decision(&self, decision: &str) { + self.span.record("openshell.policy.decision", decision); + let policy_span = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .policy_span + .take(); + if let Some(span) = policy_span { + span.record("openshell.policy.decision", decision); + } + } + + pub(crate) fn evaluate_policy(&self, operation: impl FnOnce() -> T) -> T { + let span = tracing::info_span!( + parent: &self.span, + "openshell.policy.evaluate", + "openshell.policy.decision" = tracing::field::Empty, + ); + let result = span.in_scope(operation); + self.state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .policy_span = Some(span); + result + } + + pub(crate) fn record_enforcement_action(&self, action: &str) { + self.span.record("openshell.enforcement.action", action); + } + + pub(crate) fn record_final_http_outcome(&self, outcome: &'static str) { + self.span.record("openshell.http.outcome", outcome); + self.state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .outcome = Some(outcome); + } + + pub(crate) fn record_response_status(&self, status: u16) { + self.span.record("http.response.status_code", status); + self.state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .status = Some(status); + } + + pub(crate) fn record_upstream_status(&self, status: u16) { + self.state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .upstream_status = Some(status); + } + + pub(crate) fn record_target_reached(&self) { + self.span.record("openshell.target.reached", true); + } + + pub(crate) fn record_error_stage(&self, _stage: &str) { + openshell_otel::mark_error(&self.span); + } + + pub(crate) fn emit(&self, event: OcsfEvent) { + if matches!(event, OcsfEvent::HttpActivity(_)) { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(primary) = &mut state.primary { + *primary = event; + } + } else { + self.emit_correlated(event); + } + } + + pub(crate) fn response_selected(&self, status: u16) { + self.record_response_status(status); + self.emit_primary(false); + } + + fn emit_primary(&self, incomplete: bool) { + let (mut event, status, upstream_status, outcome) = { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let Some(primary) = state.primary.take() else { + return; + }; + (primary, state.status, state.upstream_status, state.outcome) + }; + if let OcsfEvent::HttpActivity(http) = &mut event { + http.http_response = status.map(|code| HttpResponse { code }); + if let Some(upstream) = upstream_status.filter(|upstream| Some(*upstream) != status) { + let unmapped = http + .base + .unmapped + .get_or_insert_with(|| serde_json::json!({})); + unmapped["upstream_response_code"] = serde_json::json!(upstream); + } + if incomplete && outcome.is_none() { + http.base.status = Some(StatusId::Failure); + http.base.status_detail = Some("request_incomplete".into()); + http.base.severity = SeverityId::Low; + } + if outcome == Some("denied") { + if http.action != Some(ActionId::Denied) { + http.base.message = Some("HTTP request blocked after policy admission".into()); + } + http.action = Some(ActionId::Denied); + http.disposition = Some(openshell_ocsf::DispositionId::Blocked); + http.base.status = Some(StatusId::Failure); + http.base.severity = SeverityId::Medium; + } + } + self.emit_correlated(event); + } + + fn emit_correlated(&self, mut event: OcsfEvent) { + event.set_correlation_uid(self.request_id.clone()); + self.span + .in_scope(|| crate::telemetry::emit_correlated(event)); + } +} + +/// Observe accepted response bytes without buffering or changing delivery. +pub struct AuditedClient<'a, C> { + inner: &'a mut C, + correlation: &'a RequestCorrelation, + head: ResponseHead, +} + +#[derive(Default)] +struct ResponseHead { + prefix: [u8; 12], + prefix_len: usize, + delimiter: u32, + complete: bool, +} + +impl ResponseHead { + fn accept(&mut self, bytes: &[u8], correlation: &RequestCorrelation) { + for &byte in bytes { + if self.complete { + break; + } + if self.prefix_len < self.prefix.len() { + self.prefix[self.prefix_len] = byte; + self.prefix_len += 1; + } + self.delimiter = (self.delimiter << 8) | u32::from(byte); + if self.delimiter != u32::from_be_bytes(*b"\r\n\r\n") { + continue; + } + let status = std::str::from_utf8(&self.prefix[9..12]) + .ok() + .and_then(|code| code.parse::().ok()); + if status.is_some_and(|code| (100..200).contains(&code) && code != 101) { + *self = Self::default(); + } else { + self.complete = true; + if let Some(status) = status { + correlation.response_selected(status); + } + } + } + } +} + +impl AsyncRead for AuditedClient<'_, C> { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + Pin::new(&mut *self.get_mut().inner).poll_read(cx, buf) + } +} + +impl AsyncWrite for AuditedClient<'_, C> { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + bytes: &[u8], + ) -> Poll> { + let this = self.get_mut(); + let result = Pin::new(&mut *this.inner).poll_write(cx, bytes); + if let Poll::Ready(Ok(accepted)) = result { + this.head.accept(&bytes[..accepted], this.correlation); + } + result + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut *self.get_mut().inner).poll_flush(cx) + } + + fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut *self.get_mut().inner).poll_shutdown(cx) + } +} + +fn bounded_http_method(method: &str) -> &'static str { + [ + "CONNECT", "DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT", "TRACE", + ] + .into_iter() + .find(|known| method.eq_ignore_ascii_case(known)) + .unwrap_or("OTHER") +} + +impl Drop for RequestCorrelation { + fn drop(&mut self) { + if self + .state + .get_mut() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .outcome + .is_none() + { + self.span.record("openshell.http.outcome", "incomplete"); + openshell_otel::mark_error(&self.span); + } + self.emit_primary(true); + } +} + +fn trace_headers(raw_header: &[u8]) -> HeaderMap { + let Some(header_end) = raw_header + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|position| position + 2) + else { + return HeaderMap::new(); + }; + + let mut headers = HeaderMap::new(); + for line in raw_header[..header_end] + .split(|byte| *byte == b'\n') + .skip(1) + { + let line = line.strip_suffix(b"\r").unwrap_or(line); + let Some(separator) = line.iter().position(|byte| *byte == b':') else { + continue; + }; + let name = trim_optional_whitespace(&line[..separator]); + if !name.eq_ignore_ascii_case(b"traceparent") { + continue; + } + let Ok(name) = http::HeaderName::from_bytes(name) else { + continue; + }; + let Ok(value) = HeaderValue::from_bytes(trim_optional_whitespace(&line[separator + 1..])) + else { + continue; + }; + if value.to_str().is_err() { + continue; + } + headers.append(name, value); + } + headers +} + +fn trim_optional_whitespace(mut value: &[u8]) -> &[u8] { + while matches!(value.first(), Some(b' ' | b'\t')) { + value = &value[1..]; + } + while matches!(value.last(), Some(b' ' | b'\t')) { + value = &value[..value.len() - 1]; + } + value +} + +#[cfg(test)] +mod tests { + use super::*; + + fn request() -> L7Request { + L7Request { + action: "GET".into(), + target: "/data".into(), + query_params: std::collections::HashMap::new(), + raw_header: b"GET /data HTTP/1.1\r\nHost: example.test\r\n\r\n".to_vec(), + body_length: crate::l7::provider::BodyLength::None, + } + } + + #[test] + fn request_ids_are_unique_canonical_uuids() { + let ctx = L7EvalContext { + host: "example.test".into(), + port: 443, + ..Default::default() + }; + let first = RequestCorrelation::new(&request(), &ctx); + let second = RequestCorrelation::new(&request(), &ctx); + + assert_ne!(first.request_id(), second.request_id()); + assert!(uuid::Uuid::parse_str(first.request_id()).is_ok()); + assert!(uuid::Uuid::parse_str(second.request_id()).is_ok()); + } + + #[test] + fn extracts_only_traceparent_and_preserves_duplicates() { + let headers = trace_headers( + b"GET / HTTP/1.1\r\nHost: example.com\r\nX-OpenShell-Request-ID: forged\r\ntraceparent: 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\r\nTraceParent: 00-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbbbb-01\r\ntracestate: vendor=value\r\n\r\n", + ); + assert_eq!(headers.get_all("traceparent").iter().count(), 2); + assert!(headers.get("tracestate").is_none()); + assert!(headers.get("x-openshell-request-id").is_none()); + } + + #[test] + fn unrelated_obs_text_does_not_discard_valid_trace_context() { + let headers = trace_headers( + b"GET / HTTP/1.1\r\ntraceparent: 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01\r\nx-observation-note: contains traceparent: but is not a field\r\n\r\n", + ); + + assert_eq!(headers.get_all("traceparent").iter().count(), 1); + } + + #[test] + fn non_ascii_traceparent_is_rejected_and_tracestate_is_ignored() { + let headers = trace_headers( + b"GET / HTTP/1.1\r\ntraceparent: \xff\r\ntracestate: vendor=value\r\n\r\n", + ); + + assert!(headers.get("traceparent").is_none()); + assert!(headers.get("tracestate").is_none()); + } + + #[test] + fn records_only_bounded_enforcement_outcome_fields() { + use tracing_subscriber::layer::SubscriberExt as _; + + let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry() + .with(openshell_otel::layer(&provider, "correlation-test")); + let ctx = L7EvalContext { + host: "example.test".into(), + port: 443, + ..Default::default() + }; + + tracing::subscriber::with_default(subscriber, || { + let correlation = RequestCorrelation::with_request_id( + &request(), + &ctx, + "00000000-0000-4000-8000-000000000001".into(), + ); + assert!(!correlation.evaluate_policy(|| false)); + correlation.record_policy_decision("deny"); + correlation.record_enforcement_action("block"); + correlation.record_final_http_outcome("denied"); + correlation.record_response_status(403); + correlation.record_error_stage("policy"); + correlation.record_target_reached(); + }); + + let spans = exporter.get_finished_spans().unwrap(); + let span = spans + .iter() + .find(|span| span.name == "openshell.http.request") + .unwrap(); + let policy = spans + .iter() + .find(|span| span.name == "openshell.policy.evaluate") + .unwrap(); + assert_eq!(policy.parent_span_id, span.span_context.span_id()); + assert_eq!(span.span_kind, opentelemetry::trace::SpanKind::Server); + let attributes = span + .attributes + .iter() + .map(|kv| (kv.key.as_str().to_string(), kv.value.to_string())) + .collect::>(); + assert_eq!(attributes["openshell.policy.decision"], "deny"); + assert_eq!(attributes["openshell.enforcement.action"], "block"); + assert_eq!(attributes["openshell.http.outcome"], "denied"); + assert_eq!(attributes["http.response.status_code"], "403"); + assert!(!attributes.contains_key("openshell.error.stage")); + assert_eq!(attributes["openshell.target.reached"], "true"); + } + + #[test] + fn custom_method_is_bounded_in_telemetry() { + use tracing_subscriber::layer::SubscriberExt as _; + + let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry() + .with(openshell_otel::layer(&provider, "correlation-test")); + let ctx = L7EvalContext { + host: "example.test".into(), + port: 443, + ..Default::default() + }; + let mut custom = request(); + custom.action = "SENTINEL-UNTRUSTED-METHOD".into(); + + tracing::subscriber::with_default(subscriber, || { + let correlation = RequestCorrelation::with_request_id( + &custom, + &ctx, + "00000000-0000-4000-8000-000000000002".into(), + ); + correlation.record_final_http_outcome("denied"); + }); + + let span = exporter + .get_finished_spans() + .unwrap() + .into_iter() + .find(|span| span.name == "openshell.http.request") + .unwrap(); + let method = span + .attributes + .iter() + .find(|attribute| attribute.key.as_str() == "http.request.method") + .expect("method attribute"); + assert_eq!(method.value.to_string(), "OTHER"); + } + + #[test] + fn unfinished_request_is_recorded_as_incomplete() { + use tracing_subscriber::layer::SubscriberExt as _; + + let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry() + .with(openshell_otel::layer(&provider, "correlation-test")); + let ctx = L7EvalContext { + host: "example.test".into(), + port: 443, + ..Default::default() + }; + + tracing::subscriber::with_default(subscriber, || { + let _correlation = RequestCorrelation::with_request_id( + &request(), + &ctx, + "00000000-0000-4000-8000-000000000002".into(), + ); + }); + + let span = exporter + .get_finished_spans() + .unwrap() + .into_iter() + .find(|span| span.name == "openshell.http.request") + .unwrap(); + let attributes = span + .attributes + .into_iter() + .map(|kv| (kv.key.as_str().to_string(), kv.value.to_string())) + .collect::>(); + assert_eq!(attributes["openshell.http.outcome"], "incomplete"); + assert!(!attributes.contains_key("openshell.error.stage")); + } + + #[test] + fn malformed_or_incomplete_header_block_has_no_trace_context() { + assert!(trace_headers(b"GET / HTTP/1.1\r\ntraceparent: value").is_empty()); + assert!(trace_headers(&[0xff, b'\r', b'\n', b'\r', b'\n']).is_empty()); + } + + #[tokio::test] + async fn primary_record_waits_for_the_final_response_head_and_is_emitted_once() { + use tokio::io::AsyncWriteExt as _; + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let _subscriber = capture.install(); + let ctx = L7EvalContext::default(); + let correlation = RequestCorrelation::new(&request(), &ctx); + let id = correlation.request_id().to_string(); + correlation.emit( + HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .http_request(HttpRequest::new("GET", ctx.ocsf_url("/data"))) + .action(ActionId::Allowed) + .build(), + ); + let mut output = Vec::new(); + { + let mut client = correlation.observe_response(&mut output); + client + .write_all(b"HTTP/1.1 100 Continue\r\n\r\nHTTP/1.1 200 OK\r\nContent-Length: 4\r\n") + .await + .unwrap(); + assert!(capture.events().is_empty()); + client.write_all(b"\r\nbody").await.unwrap(); + let events = capture.json_events(); + assert_eq!(events.len(), 1); + assert_eq!(events[0]["http_response"]["code"], 200); + assert_eq!(events[0]["metadata"]["correlation_uid"], id); + } + drop(correlation); + assert_eq!(capture.events().len(), 1); + assert!(output.ends_with(b"\r\n\r\nbody")); + } + + #[test] + fn request_ending_before_response_headers_still_has_one_primary_record() { + let capture = openshell_ocsf::test_support::OcsfCapture::default(); + let _subscriber = capture.install(); + let correlation = RequestCorrelation::new(&request(), &L7EvalContext::default()); + let id = correlation.request_id().to_string(); + let mut head = ResponseHead::default(); + head.accept(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n", &correlation); + assert!(capture.events().is_empty()); + drop(correlation); + let events = capture.json_events(); + assert_eq!(events.len(), 1); + assert_eq!(events[0]["metadata"]["correlation_uid"], id); + assert_eq!(events[0]["status_detail"], "request_incomplete"); + assert!(events[0].get("http_response").is_none()); + } +} diff --git a/crates/openshell-supervisor-network/src/l7/middleware.rs b/crates/openshell-supervisor-network/src/l7/middleware.rs index 73c317be6f..94efc3fd9c 100644 --- a/crates/openshell-supervisor-network/src/l7/middleware.rs +++ b/crates/openshell-supervisor-network/src/l7/middleware.rs @@ -668,8 +668,11 @@ pub async fn send_middleware_rejection_response, redacted_target: &str, + request_id: Option<&str>, ) -> Result<()> { - let context = Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)); + let context = crate::l7::rest::DenyResponseContext::from_l7_context(ctx); + let context = request_id.map_or(context, |request_id| context.with_request_id(request_id)); + let context = Some(context); if let Some(denial) = denial { crate::l7::rest::send_middleware_deny_response( req, @@ -699,13 +702,16 @@ pub async fn send_middleware_admission_exhausted_response< client: &mut C, ctx: &L7EvalContext, redacted_target: &str, + request_id: Option<&str>, ) -> Result<()> { + let context = crate::l7::rest::DenyResponseContext::from_l7_context(ctx); + let context = request_id.map_or(context, |request_id| context.with_request_id(request_id)); crate::l7::rest::send_middleware_unavailable_response( req, &ctx.policy_name, client, Some(redacted_target), - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some(context), ) .await } @@ -1397,9 +1403,16 @@ mod tests { }; let (mut client, mut server) = tokio::io::duplex(4096); - send_middleware_rejection_response(&req, &mut server, &ctx, Some(&denial), "/v1/messages") - .await - .expect("send denial"); + send_middleware_rejection_response( + &req, + &mut server, + &ctx, + Some(&denial), + "/v1/messages", + Some("00000000-0000-4000-8000-000000000010"), + ) + .await + .expect("send denial"); drop(server); let mut response = Vec::new(); @@ -1408,9 +1421,13 @@ mod tests { .await .expect("read denial"); let response = String::from_utf8(response).expect("UTF-8 response"); + assert!( + response.contains("x-openshell-request-id: 00000000-0000-4000-8000-000000000010\r\n") + ); let (_, body) = response.split_once("\r\n\r\n").expect("HTTP response"); let body: serde_json::Value = serde_json::from_str(body).expect("JSON response"); assert_eq!(body["error"], "middleware_denied"); + assert_eq!(body["request_id"], "00000000-0000-4000-8000-000000000010"); assert_eq!(body["middleware"], "prototype-content-guard"); assert_eq!(body["reason_code"], "content_match"); assert!(body.get("rule_missing").is_none()); @@ -1490,9 +1507,16 @@ mod tests { }; let (mut client, mut server) = tokio::io::duplex(4096); - send_middleware_rejection_response(&req, &mut server, &ctx, None, "/v1/messages") - .await - .expect("send failure"); + send_middleware_rejection_response( + &req, + &mut server, + &ctx, + None, + "/v1/messages", + Some("00000000-0000-4000-8000-000000000011"), + ) + .await + .expect("send failure"); drop(server); let mut response = Vec::new(); @@ -1501,9 +1525,13 @@ mod tests { .await .expect("read failure"); let response = String::from_utf8(response).expect("UTF-8 response"); + assert!( + response.contains("x-openshell-request-id: 00000000-0000-4000-8000-000000000011\r\n") + ); let (_, body) = response.split_once("\r\n\r\n").expect("HTTP response"); let body: serde_json::Value = serde_json::from_str(body).expect("JSON response"); assert_eq!(body["error"], "middleware_failed"); + assert_eq!(body["request_id"], "00000000-0000-4000-8000-000000000011"); assert_eq!( body["detail"], "Request could not be processed by configured middleware" diff --git a/crates/openshell-supervisor-network/src/l7/mod.rs b/crates/openshell-supervisor-network/src/l7/mod.rs index bc01d0bbdd..687ba9846a 100644 --- a/crates/openshell-supervisor-network/src/l7/mod.rs +++ b/crates/openshell-supervisor-network/src/l7/mod.rs @@ -8,6 +8,7 @@ //! doing a raw `copy_bidirectional`. Each request within the tunnel is parsed, //! evaluated against OPA policy, and either forwarded or denied. +pub(crate) mod correlation; pub mod graphql; pub(crate) mod http; pub mod jsonrpc; diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index fa20252714..ad2ef43b33 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -7,6 +7,7 @@ //! Parses each request within the tunnel, evaluates it against OPA policy, //! and either forwards or denies the request. +use crate::l7::correlation::RequestCorrelation; use crate::l7::middleware::{ MiddlewareApplyResult, UninspectableTrafficGate, apply_middleware_chain_with_request_id, emit_middleware_uninspectable, middleware_network_input, uninspectable_traffic_gate, @@ -36,6 +37,8 @@ use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; use tracing::{debug, warn}; +#[cfg(test)] +mod correlation_dispatch_tests; #[cfg(test)] mod token_grant_ownership_tests; @@ -272,35 +275,50 @@ async fn reject_request_authority_mismatch( client: &mut W, ctx: &L7EvalContext, method: &str, + correlation: Option<&RequestCorrelation>, ) -> Result<()> where W: AsyncWrite + Unpin, { - let body = r#"{"error":"request_authority_mismatch","message":"HTTP request authority does not match the authorized tunnel endpoint"}"#; - let response = format!( - "HTTP/1.1 403 Forbidden\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", - body.len() - ); - client - .write_all(response.as_bytes()) - .await - .into_diagnostic()?; - client.flush().await.into_diagnostic()?; - - crate::telemetry::emit_decision( - [ - build_request_authority_mismatch_event(ctx, method), - build_request_authority_mismatch_finding(ctx), - ], - None, - ); + let activity = build_request_authority_mismatch_event(ctx, method); + let finding = build_request_authority_mismatch_finding(ctx); + if let Some(correlation) = correlation { + correlation.record_policy_decision("deny"); + correlation.record_enforcement_action("block"); + correlation.record_response_status(403); + correlation.record_final_http_outcome("denied"); + correlation.record_error_stage("request_authority"); + correlation.emit(activity); + correlation.emit(finding); + crate::l7::rest::send_json_response_with_request_id( + &ctx.policy_name, + serde_json::json!({ + "error": "request_authority_mismatch", + "message": "HTTP request authority does not match the authorized tunnel endpoint", + }), + client, + "403 Forbidden", + correlation.request_id(), + ) + .await?; + } else { + crate::telemetry::emit_decision([activity, finding], None); + crate::l7::rest::send_json_response( + &ctx.policy_name, + serde_json::json!({ + "error": "request_authority_mismatch", + "message": "HTTP request authority does not match the authorized tunnel endpoint", + }), + client, + "403 Forbidden", + ) + .await?; + } Ok(()) } -/// Return the selected revision's inspection for policy evaluation, or emit a -/// rejection and return `None`. Non-MCP adapters retain their original inspection. -/// Record local rejections before client response delivery can fail. -pub(crate) async fn enforce_mcp_protocol_version( +#[allow(clippy::too_many_arguments)] +pub(crate) async fn enforce_mcp_protocol_version_correlated( config: &L7EndpointConfig, request: &crate::l7::provider::L7Request, mut info: crate::l7::jsonrpc::JsonRpcRequestInfo, @@ -308,6 +326,7 @@ pub(crate) async fn enforce_mcp_protocol_version( ctx: &L7EvalContext, redacted_target: &str, observer: Option<&EndpointObserver>, + correlation: Option<&RequestCorrelation>, ) -> Result> where W: AsyncWrite + Unpin, @@ -343,7 +362,7 @@ where crate::l7::mcp::selected_revision_context(request, version) ); let summary = l7_protocol_log_summary(None, Some(&info)); - egress_emit!(build_l7_request_event( + let event = build_l7_request_event( ctx, &request.action, redacted_target, @@ -351,7 +370,17 @@ where "l7-mcp", &reason, summary.as_deref(), - )); + ); + if let Some(correlation) = correlation { + correlation.record_policy_decision("deny"); + correlation.record_enforcement_action("block"); + correlation.record_response_status(400); + correlation.record_final_http_outcome("denied"); + correlation.record_error_stage("mcp_validation"); + correlation.emit(event); + } else { + egress_emit!(event); + } emit_activity(ctx, true, "l7_parse_rejection"); let body = serde_json::json!({ "error": "invalid_mcp_request", @@ -362,13 +391,24 @@ where "method": request.action, "path": redacted_target, }); - crate::l7::rest::send_json_response( - &ctx.policy_name, - body, - client, - "400 Bad Request", - ) - .await?; + if let Some(correlation) = correlation { + crate::l7::rest::send_json_response_with_request_id( + &ctx.policy_name, + body, + client, + "400 Bad Request", + correlation.request_id(), + ) + .await?; + } else { + crate::l7::rest::send_json_response( + &ctx.policy_name, + body, + client, + "400 Bad Request", + ) + .await?; + } return Ok(None); } @@ -381,6 +421,7 @@ where ctx, redacted_target, observer, + correlation, ) .await?; return Ok(None); @@ -397,6 +438,7 @@ where ctx, redacted_target, observer, + correlation, ) .await?; Ok(None) @@ -405,6 +447,7 @@ where } /// Emit the same transport or policy rejection at initial and final inspection. +#[allow(clippy::too_many_arguments)] async fn reject_mcp_protocol_version( error: crate::l7::mcp::McpProtocolVersionError, request: &crate::l7::provider::L7Request, @@ -413,6 +456,7 @@ async fn reject_mcp_protocol_version( ctx: &L7EvalContext, redacted_target: &str, observer: Option<&EndpointObserver>, + correlation: Option<&RequestCorrelation>, ) -> Result<()> where W: AsyncWrite + Unpin, @@ -424,7 +468,7 @@ where } let reason = error.rejection_detail(request); let summary = l7_protocol_log_summary(None, Some(info)); - egress_emit!(build_l7_request_event( + let event = build_l7_request_event( ctx, &request.action, redacted_target, @@ -432,7 +476,23 @@ where "l7-mcp", &reason, summary.as_deref(), - )); + ); + if let Some(correlation) = correlation { + correlation.record_policy_decision("deny"); + correlation.record_enforcement_action("block"); + correlation.record_response_status(match error { + crate::l7::mcp::McpProtocolVersionError::InvalidHeader + | crate::l7::mcp::McpProtocolVersionError::UnsupportedHeaderValue + | crate::l7::mcp::McpProtocolVersionError::InvalidRequestMetadata => 400, + crate::l7::mcp::McpProtocolVersionError::MethodNotAllowed => 405, + crate::l7::mcp::McpProtocolVersionError::NotAllowed(_) => 403, + }); + correlation.record_final_http_outcome("denied"); + correlation.record_error_stage("mcp_protocol"); + correlation.emit(event); + } else { + egress_emit!(event); + } let deny_group = match error { crate::l7::mcp::McpProtocolVersionError::NotAllowed(_) => "l7_policy", crate::l7::mcp::McpProtocolVersionError::InvalidHeader @@ -452,19 +512,32 @@ where }); let allowed_methods = (error == crate::l7::mcp::McpProtocolVersionError::MethodNotAllowed).then_some("POST"); - crate::l7::rest::send_json_response_with_allow( - &ctx.policy_name, - body, - client, - error.http_status(), - allowed_methods, - ) - .await?; + if let Some(correlation) = correlation { + crate::l7::rest::send_json_response_with_allow_and_request_id( + &ctx.policy_name, + body, + client, + error.http_status(), + allowed_methods, + correlation.request_id(), + ) + .await?; + } else { + crate::l7::rest::send_json_response_with_allow( + &ctx.policy_name, + body, + client, + error.http_status(), + allowed_methods, + ) + .await?; + } Ok(()) } /// Reinspect the buffered outgoing MCP request after request transformations. /// The forwarding adapter must call this before any upstream request write. +#[cfg(test)] pub(crate) async fn enforce_final_mcp_protocol_version( config: &L7EndpointConfig, request: &crate::l7::provider::L7Request, @@ -473,6 +546,31 @@ pub(crate) async fn enforce_final_mcp_protocol_version( redacted_target: &str, observer: Option<&EndpointObserver>, ) -> Result +where + W: AsyncWrite + Unpin, +{ + enforce_final_mcp_protocol_version_correlated( + config, + request, + client, + ctx, + redacted_target, + observer, + None, + ) + .await +} + +#[allow(clippy::too_many_arguments)] +pub(crate) async fn enforce_final_mcp_protocol_version_correlated( + config: &L7EndpointConfig, + request: &crate::l7::provider::L7Request, + client: &mut W, + ctx: &L7EvalContext, + redacted_target: &str, + observer: Option<&EndpointObserver>, + correlation: Option<&RequestCorrelation>, +) -> Result where W: AsyncWrite + Unpin, { @@ -483,7 +581,7 @@ where request, crate::l7::jsonrpc::JsonRpcInspectionOptions::for_config(config), )?; - Ok(enforce_mcp_protocol_version( + Ok(enforce_mcp_protocol_version_correlated( config, request, info, @@ -491,6 +589,7 @@ where ctx, redacted_target, observer, + correlation, ) .await? .is_some()) @@ -593,6 +692,7 @@ pub(crate) async fn reject_credential_resolution( ctx: &L7EvalContext, method: &str, error: &secrets::UnresolvedPlaceholderError, + correlation: Option<&RequestCorrelation>, ) -> Result<()> where W: AsyncWrite + Unpin, @@ -605,22 +705,21 @@ where "500 Internal Server Error" }; let body = if placement_mismatch { - r#"{"error":"credential_placement_mismatch","message":"Credential is not authorized for this request location"}"# + serde_json::json!({ + "error": "credential_placement_mismatch", + "message": "Credential is not authorized for this request location", + }) } else if endpoint_mismatch { - r#"{"error":"credential_endpoint_mismatch","message":"Credential is not authorized for this request endpoint"}"# + serde_json::json!({ + "error": "credential_endpoint_mismatch", + "message": "Credential is not authorized for this request endpoint", + }) } else { - r#"{"error":"credential_unavailable","message":"Credential placeholder could not be resolved"}"# + serde_json::json!({ + "error": "credential_unavailable", + "message": "Credential placeholder could not be resolved", + }) }; - let response = format!( - "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", - body.len() - ); - client - .write_all(response.as_bytes()) - .await - .into_diagnostic()?; - client.flush().await.into_diagnostic()?; - let mut events = if placement_mismatch { vec![ HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -652,28 +751,102 @@ where if endpoint_mismatch { events.push(build_credential_endpoint_mismatch_finding(ctx)); } - crate::telemetry::emit_decision(events, None); + if let Some(correlation) = correlation { + correlation.record_enforcement_action("block"); + correlation.record_response_status(if endpoint_mismatch || placement_mismatch { + 403 + } else { + 500 + }); + correlation.record_final_http_outcome(if endpoint_mismatch || placement_mismatch { + "denied" + } else { + "platform_error" + }); + correlation.record_error_stage("credential_resolution"); + for event in events { + correlation.emit(event); + } + crate::l7::rest::send_json_response_with_request_id( + &ctx.policy_name, + body, + client, + status, + correlation.request_id(), + ) + .await?; + } else { + crate::telemetry::emit_decision(events, None); + crate::l7::rest::send_json_response(&ctx.policy_name, body, client, status).await?; + } Ok(()) } pub(crate) async fn reject_body_credential( client: &mut C, error: secrets::body::BodyCredentialError, + ctx: &L7EvalContext, + correlation: Option<&RequestCorrelation>, ) -> Result<()> { let body = serde_json::json!({"error": { "code": "credential_placeholder_in_request_body", "reason": error.reason(), "message": "A credential placeholder in the request body cannot be forwarded. Remove the reference from conversation history or restore provider access; body credential rewriting is disabled." - }}).to_string(); - let response = format!( - "HTTP/1.1 403 Forbidden\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", - body.len() - ); - client - .write_all(response.as_bytes()) + }}); + if let Some(correlation) = correlation { + correlation.record_policy_decision("deny"); + correlation.record_enforcement_action("block"); + correlation.record_response_status(403); + correlation.record_final_http_outcome("denied"); + correlation.record_error_stage("credential_body"); + crate::l7::rest::send_json_response_with_request_id( + &ctx.policy_name, + body, + client, + "403 Forbidden", + correlation.request_id(), + ) .await - .into_diagnostic()?; - client.flush().await.into_diagnostic() + } else { + crate::l7::rest::send_json_response(&ctx.policy_name, body, client, "403 Forbidden").await + } +} + +async fn reject_credential_unavailable( + client: &mut C, + ctx: &L7EvalContext, + method: &str, + correlation: Option<&RequestCorrelation>, +) -> Result<()> { + let event = build_credential_resolution_event(ctx, method, false); + let body = serde_json::json!({ + "error": "credential_unavailable", + "message": "Credential material is unavailable for this request", + }); + if let Some(correlation) = correlation { + correlation.record_enforcement_action("block"); + correlation.record_response_status(500); + correlation.record_final_http_outcome("platform_error"); + correlation.record_error_stage("credential_availability"); + correlation.emit(event); + crate::l7::rest::send_json_response_with_request_id( + &ctx.policy_name, + body, + client, + "500 Internal Server Error", + correlation.request_id(), + ) + .await + } else { + egress_emit!(event); + crate::l7::rest::send_json_response( + &ctx.policy_name, + body, + client, + "500 Internal Server Error", + ) + .await + } } struct InspectedForwarding<'a> { @@ -681,6 +854,7 @@ struct InspectedForwarding<'a> { engine: &'a TunnelPolicyEngine, request_info: &'a L7RequestInfo, request_id: &'a str, + correlation: Option<&'a RequestCorrelation>, response_chain: &'a [openshell_supervisor_middleware::ChainEntry], websocket_middleware: bool, observation_context: Option<&'a openshell_core::endpoint_status::EndpointObservationContext>, @@ -705,6 +879,7 @@ where engine, request_info, request_id, + correlation, response_chain, websocket_middleware, observation_context, @@ -722,6 +897,12 @@ where { Ok(prepared) => prepared, Err(error) => { + if let Some(correlation) = correlation { + correlation.record_enforcement_action("block"); + correlation.record_response_status(502); + correlation.record_final_http_outcome("platform_error"); + correlation.record_error_stage("credential_preparation"); + } if let Some(refusal) = error.downcast_ref::() { @@ -740,7 +921,8 @@ where write_bad_gateway_response(client, Some(request_id)).await?; } else { warn!(error = %error, "Token grant failed before forwarding"); - write_bad_gateway_response(client, None).await?; + write_bad_gateway_response(client, correlation.map(RequestCorrelation::request_id)) + .await?; } return Ok(None); } @@ -790,7 +972,10 @@ where engine.middleware_runner(), Some(engine.generation_guard()), )), - observer.as_ref(), + RelayHttpInstrumentation { + observer: observer.as_ref(), + correlation, + }, ) .await } @@ -814,11 +999,17 @@ where options, ctx, response_middleware, - None, + RelayHttpInstrumentation::default(), ) .await } +#[derive(Clone, Copy, Default)] +struct RelayHttpInstrumentation<'a> { + observer: Option<&'a EndpointObserver>, + correlation: Option<&'a RequestCorrelation>, +} + async fn relay_http_request_with_credential_rejection_observed( request: &crate::l7::provider::L7Request, client: &mut C, @@ -826,20 +1017,25 @@ async fn relay_http_request_with_credential_rejection_observed( options: crate::l7::rest::RelayRequestOptions<'_>, ctx: &L7EvalContext, response_middleware: Option>, - observer: Option<&EndpointObserver>, + instrumentation: RelayHttpInstrumentation<'_>, ) -> Result> where C: AsyncRead + AsyncWrite + Unpin, U: AsyncRead + AsyncWrite + Unpin, { + let RelayHttpInstrumentation { + observer, + correlation, + } = instrumentation; match Box::pin( - crate::l7::rest::relay_http_request_with_response_middleware_guarded_observed( + crate::l7::rest::relay_http_request_with_response_middleware_guarded_observed_correlated( request, client, upstream, options, response_middleware, observer, + correlation, ), ) .await @@ -854,23 +1050,32 @@ where observer.observe(EndpointResult::PolicyDenied); } let _ = upstream.shutdown().await; - reject_body_credential(client, *error).await?; + reject_body_credential(client, *error, ctx, correlation).await?; Ok(None) } else if let Some(error) = report.downcast_ref::() { if let Some(observer) = observer { observer.observe_credential_failure(error.is_endpoint_mismatch()); } - reject_credential_resolution(client, ctx, &request.action, error).await?; + reject_credential_resolution(client, ctx, &request.action, error, correlation) + .await?; Ok(None) - } else if report - .downcast_ref::() - .is_some() + } else if let Some(error) = + report.downcast_ref::() { if let Some(observer) = observer { observer.observe_credential_failure(false); } - Err(report) + if error.generation_is_stale() { + if let Some(correlation) = correlation { + correlation.record_enforcement_action("block"); + correlation.record_final_http_outcome("platform_error"); + correlation.record_error_stage("credential_generation"); + } + return Err(report); + } + reject_credential_unavailable(client, ctx, &request.action, correlation).await?; + Ok(None) } else { Err(report) } @@ -1027,6 +1232,15 @@ fn parse_rejection_detail(error: &str, mode: ParseRejectionMode) -> String { } fn emit_parse_rejection(ctx: &L7EvalContext, detail: &str, engine_type: &str) { + emit_parse_rejection_correlated(ctx, detail, engine_type, None); +} + +fn emit_parse_rejection_correlated( + ctx: &L7EvalContext, + detail: &str, + engine_type: &str, + correlation: Option<&RequestCorrelation>, +) { let policy_name = if ctx.policy_name.is_empty() { "-" } else { @@ -1046,7 +1260,11 @@ fn emit_parse_rejection(ctx: &L7EvalContext, detail: &str, engine_type: &str) { )) .status_detail(detail) .build(); - egress_emit!(event); + if let Some(correlation) = correlation { + correlation.emit(event); + } else { + egress_emit!(event); + } emit_activity(ctx, true, "l7_parse_rejection"); } @@ -1075,6 +1293,7 @@ async fn deny_unsupported_upgrade_if_requested( ctx: &L7EvalContext, observer: Option<&EndpointObserver>, client: &mut C, + correlation: Option<&RequestCorrelation>, ) -> Result where C: AsyncRead + AsyncWrite + Unpin + Send, @@ -1088,17 +1307,36 @@ where if let Some(observer) = observer { observer.observe(EndpointResult::PolicyDenied); } - emit_parse_rejection(ctx, detail, engine_type_for_protocol(config.protocol)); - crate::l7::rest::send_json_response( - &ctx.policy_name, - serde_json::json!({ - "error": "unsupported_l7_protocol", - "detail": detail, - }), - client, - "403 Forbidden", - ) - .await?; + if let Some(correlation) = correlation { + correlation.record_policy_decision("deny"); + correlation.record_enforcement_action("block"); + correlation.record_response_status(403); + correlation.record_final_http_outcome("denied"); + correlation.record_error_stage("protocol_upgrade"); + } + emit_parse_rejection_correlated( + ctx, + detail, + engine_type_for_protocol(config.protocol), + correlation, + ); + let body = serde_json::json!({ + "error": "unsupported_l7_protocol", + "detail": detail, + }); + if let Some(correlation) = correlation { + crate::l7::rest::send_json_response_with_request_id( + &ctx.policy_name, + body, + client, + "403 Forbidden", + correlation.request_id(), + ) + .await?; + } else { + crate::l7::rest::send_json_response(&ctx.policy_name, body, client, "403 Forbidden") + .await?; + } Ok(true) } @@ -1235,21 +1473,38 @@ where return Ok(()); } }; + let correlation = RequestCorrelation::new(&req, ctx); + let mut audited_client = correlation.observe_response(client); + let client = &mut audited_client; if !request_authority_matches_endpoint(&req, ctx) { - reject_request_authority_mismatch(client, ctx, &req.action).await?; + reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)).await?; return Ok(()); } let route_target = match secrets::redact_target_for_policy(&req.target) { Ok(target) => target, Err(error) => { - reject_credential_resolution(client, ctx, &req.action, &error).await?; + reject_credential_resolution(client, ctx, &req.action, &error, Some(&correlation)) + .await?; return Ok(()); } }; let Some(config) = select_l7_config_for_path(configs, &route_target) else { let reason = "no L7 endpoint path matched request"; - emit_l7_request_log(ctx, &req.action, &route_target, "deny", "l7", reason, None); + correlation.record_policy_decision("deny"); + correlation.record_enforcement_action("block"); + correlation.record_response_status(403); + correlation.record_final_http_outcome("denied"); + emit_l7_request_log_correlated( + ctx, + &req.action, + &route_target, + "deny", + "l7", + reason, + None, + Some(&correlation), + ); crate::l7::rest::RestProvider::default() .deny_with_redacted_target( &req, @@ -1257,7 +1512,10 @@ where reason, client, None, - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some( + crate::l7::rest::DenyResponseContext::from_l7_context(ctx) + .with_request_id(correlation.request_id()), + ), ) .await?; return Ok(()); @@ -1285,7 +1543,23 @@ where if let Some(observer) = observer.as_ref() { observer.observe(EndpointResult::PolicyDenied); } - emit_parse_rejection(ctx, detail, engine_type_for_protocol(config.protocol)); + emit_parse_rejection_correlated( + ctx, + detail, + engine_type_for_protocol(config.protocol), + Some(&correlation), + ); + { + correlation.record_policy_decision("deny"); + correlation.record_enforcement_action("block"); + correlation.record_response_status(403); + correlation.record_final_http_outcome("denied"); + correlation.record_error_stage("request_target"); + } + let deny_context = crate::l7::rest::DenyResponseContext::from_l7_context(ctx); + let deny_context = Some(&correlation).map_or(deny_context, |correlation| { + deny_context.with_request_id(correlation.request_id()) + }); crate::l7::rest::RestProvider::default() .deny_with_redacted_target( &req, @@ -1293,13 +1567,20 @@ where detail, client, None, - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some(deny_context), ) .await?; return Ok(()); } - if deny_unsupported_upgrade_if_requested(&req, config, ctx, observer.as_ref(), client) - .await? + if deny_unsupported_upgrade_if_requested( + &req, + config, + ctx, + observer.as_ref(), + client, + Some(&correlation), + ) + .await? { return Ok(()); } @@ -1324,7 +1605,18 @@ where } else { let detail = parse_rejection_detail(&e.to_string(), ParseRejectionMode::L7Endpoint); - emit_parse_rejection(ctx, &detail, "l7-graphql"); + { + correlation.record_policy_decision("deny"); + correlation.record_enforcement_action("block"); + correlation.record_final_http_outcome("denied"); + correlation.record_error_stage("request_body"); + } + emit_parse_rejection_correlated( + ctx, + &detail, + "l7-graphql", + Some(&correlation), + ); } return Ok(()); } @@ -1360,7 +1652,18 @@ where &e.to_string(), ParseRejectionMode::L7Endpoint, ); - emit_parse_rejection(ctx, &detail, "l7-jsonrpc"); + { + correlation.record_policy_decision("deny"); + correlation.record_enforcement_action("block"); + correlation.record_final_http_outcome("denied"); + correlation.record_error_stage("request_body"); + } + emit_parse_rejection_correlated( + ctx, + &detail, + "l7-jsonrpc", + Some(&correlation), + ); } return Ok(()); } @@ -1380,13 +1683,14 @@ where if let Some(observer) = observer.as_ref() { observer.observe_credential_failure(error.is_endpoint_mismatch()); } - reject_credential_resolution(client, ctx, &req.action, &error).await?; + reject_credential_resolution(client, ctx, &req.action, &error, Some(&correlation)) + .await?; return Ok(()); } }; if let Some(info) = jsonrpc_info.take() { - let Some(inspected) = enforce_mcp_protocol_version( + let Some(inspected) = enforce_mcp_protocol_version_correlated( config, &req, info, @@ -1394,6 +1698,7 @@ where ctx, &redacted_target, observer.as_ref(), + Some(&correlation), ) .await? else { @@ -1410,14 +1715,36 @@ where }; let websocket_request = crate::l7::rest::request_is_websocket_upgrade(&req.raw_header); if config.protocol == L7Protocol::Websocket && !websocket_request { + let reason = "websocket endpoint requires a valid WebSocket upgrade request"; + { + correlation.record_policy_decision("deny"); + correlation.record_enforcement_action("block"); + correlation.record_response_status(403); + correlation.record_final_http_outcome("denied"); + correlation.record_error_stage("websocket_upgrade"); + } + emit_l7_request_log_correlated( + ctx, + &request_info.action, + &redacted_target, + "deny", + "l7-websocket", + reason, + None, + Some(&correlation), + ); + let deny_context = crate::l7::rest::DenyResponseContext::from_l7_context(ctx); + let deny_context = Some(&correlation).map_or(deny_context, |correlation| { + deny_context.with_request_id(correlation.request_id()) + }); crate::l7::rest::RestProvider::default() .deny_with_redacted_target( &req, &ctx.policy_name, - "websocket endpoint requires a valid WebSocket upgrade request", + reason, client, Some(&redacted_target), - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some(deny_context), ) .await?; return Ok(()); @@ -1428,7 +1755,7 @@ where let (allowed, reason) = if let Some(reason) = hard_deny_reason { (false, reason) } else { - evaluate_l7_request(&engine, ctx, &request_info)? + correlation.evaluate_policy(|| evaluate_l7_request(&engine, ctx, &request_info))? }; if close_if_stale(engine.generation_guard(), ctx) { @@ -1441,10 +1768,20 @@ where (false, EnforcementMode::Audit) => "audit", (false, EnforcementMode::Enforce) => "deny", }; + { + correlation.record_policy_decision(if allowed { "allow" } else { "deny" }); + correlation.record_enforcement_action(if allowed { + "allow" + } else if config.enforcement == EnforcementMode::Audit && !force_deny { + "audit" + } else { + "block" + }); + } let engine_type = engine_type_for_protocol(config.protocol); let protocol_summary = l7_protocol_log_summary(graphql_info.as_ref(), jsonrpc_info.as_ref()); - emit_l7_request_log( + emit_l7_request_log_correlated( ctx, &request_info.action, &redacted_target, @@ -1452,12 +1789,13 @@ where engine_type, &reason, protocol_summary.as_deref(), + Some(&correlation), ); if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; let response_chain = chain.clone(); - let request_id = uuid::Uuid::new_v4().to_string(); + let request_id = correlation.request_id(); let websocket_chain = websocket_request.then(|| chain.clone()); // Route selection resolved `config` per request, so re-check the // body against that protocol's policy after every transforming @@ -1472,7 +1810,7 @@ where engine.middleware_runner(), engine.generation_guard(), openshell_supervisor_middleware::TransformedBodyPolicy::Reevaluate(&validate), - &request_id, + request_id, ) .await; let req = match middleware_result? { @@ -1481,6 +1819,16 @@ where if let Some(observer) = observer.as_ref() { observer.observe(EndpointResult::PolicyDenied); } + { + correlation.record_enforcement_action("block"); + correlation.record_response_status(403); + correlation.record_final_http_outcome(if denial.is_some() { + "denied" + } else { + "platform_error" + }); + correlation.record_error_stage("request_middleware"); + } let denied_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -1494,11 +1842,18 @@ where ctx, denial.as_ref(), &redacted_target, + Some(correlation.request_id()), ) .await?; return Ok(()); } MiddlewareApplyResult::AdmissionExhausted => { + { + correlation.record_enforcement_action("block"); + correlation.record_response_status(503); + correlation.record_final_http_outcome("platform_error"); + correlation.record_error_stage("middleware_admission"); + } let unavailable_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -1511,6 +1866,7 @@ where client, ctx, &redacted_target, + Some(correlation.request_id()), ) .await?; return Ok(()); @@ -1529,18 +1885,35 @@ where Ok(preflight) => preflight, Err(error) => { warn!(error = %error, "WebSocket middleware preflight failed"); - write_bad_gateway_response(client, None).await?; + { + correlation.record_enforcement_action("block"); + correlation.record_response_status(502); + correlation.record_final_http_outcome("platform_error"); + correlation.record_error_stage("websocket_middleware"); + } + write_bad_gateway_response(client, Some(correlation.request_id())).await?; return Ok(()); } }; crate::l7::middleware::emit_websocket_preflight_events(ctx, &preflight); if preflight.terminal_reason.is_some() { + { + correlation.record_enforcement_action("block"); + correlation.record_response_status(403); + correlation.record_final_http_outcome(if preflight.denial.is_some() { + "denied" + } else { + "platform_error" + }); + correlation.record_error_stage("websocket_middleware"); + } crate::l7::middleware::send_middleware_rejection_response( &req, client, ctx, preflight.denial.as_ref(), &redacted_target, + Some(correlation.request_id()), ) .await?; return Ok(()); @@ -1560,7 +1933,8 @@ where config, engine: &engine, request_info: &request_info, - request_id: &request_id, + request_id, + correlation: Some(&correlation), response_chain: &response_chain, websocket_middleware: middleware_session.is_some(), observation_context: observation_context.as_ref(), @@ -1652,6 +2026,14 @@ where if let Some(observer) = observer.as_ref() { observer.observe(EndpointResult::PolicyDenied); } + { + correlation.record_response_status(403); + correlation.record_final_http_outcome("denied"); + } + let deny_context = crate::l7::rest::DenyResponseContext::from_l7_context(ctx); + let deny_context = Some(&correlation).map_or(deny_context, |correlation| { + deny_context.with_request_id(correlation.request_id()) + }); crate::l7::rest::RestProvider::default() .deny_with_redacted_target( &req, @@ -1659,7 +2041,7 @@ where &reason, client, Some(&redacted_target), - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some(deny_context), ) .await?; return Ok(()); @@ -1677,7 +2059,8 @@ fn select_l7_config_for_path<'a>( .max_by_key(|config| config.path_specificity()) } -fn emit_l7_request_log( +#[allow(clippy::too_many_arguments)] +fn emit_l7_request_log_correlated( ctx: &L7EvalContext, action: &str, redacted_target: &str, @@ -1685,6 +2068,7 @@ fn emit_l7_request_log( engine_type: &str, reason: &str, protocol_summary: Option<&str>, + correlation: Option<&RequestCorrelation>, ) { let event = build_l7_request_event( ctx, @@ -1695,7 +2079,11 @@ fn emit_l7_request_log( reason, protocol_summary, ); - egress_emit!(event); + if let Some(correlation) = correlation { + correlation.emit(event); + } else { + egress_emit!(event); + } emit_activity(ctx, decision_str == "deny", "l7_policy"); } @@ -2101,11 +2489,23 @@ where return Ok(()); // Close connection on parse error } }; + let correlation = RequestCorrelation::new(&req, ctx); + let mut audited_client = correlation.observe_response(client); + let client = &mut audited_client; if !request_authority_matches_endpoint(&req, ctx) { - reject_request_authority_mismatch(client, ctx, &req.action).await?; + reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)).await?; return Ok(()); } - if deny_unsupported_upgrade_if_requested(&req, config, ctx, None, client).await? { + if deny_unsupported_upgrade_if_requested( + &req, + config, + ctx, + None, + client, + Some(&correlation), + ) + .await? + { return Ok(()); } @@ -2118,7 +2518,8 @@ where let redacted_target = match secrets::redact_target_for_policy(&req.target) { Ok(target) => target, Err(error) => { - reject_credential_resolution(client, ctx, &req.action, &error).await?; + reject_credential_resolution(client, ctx, &req.action, &error, Some(&correlation)) + .await?; return Ok(()); } }; @@ -2132,21 +2533,41 @@ where }; let websocket_request = crate::l7::rest::request_is_websocket_upgrade(&req.raw_header); if config.protocol == L7Protocol::Websocket && !websocket_request { + let reason = "websocket endpoint requires a valid WebSocket upgrade request"; + correlation.record_policy_decision("deny"); + correlation.record_enforcement_action("block"); + correlation.record_response_status(403); + correlation.record_final_http_outcome("denied"); + correlation.record_error_stage("websocket_upgrade"); + emit_l7_request_log_correlated( + ctx, + &request_info.action, + &redacted_target, + "deny", + "l7-websocket", + reason, + None, + Some(&correlation), + ); provider .deny_with_redacted_target( &req, &ctx.policy_name, - "websocket endpoint requires a valid WebSocket upgrade request", + reason, client, Some(&redacted_target), - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some( + crate::l7::rest::DenyResponseContext::from_l7_context(ctx) + .with_request_id(correlation.request_id()), + ), ) .await?; return Ok(()); } // Evaluate L7 policy via Rego (using redacted target) - let (allowed, reason) = evaluate_l7_request(engine, ctx, &request_info)?; + let (allowed, reason) = + correlation.evaluate_policy(|| evaluate_l7_request(engine, ctx, &request_info))?; if close_if_stale(engine.generation_guard(), ctx) { return Ok(()); @@ -2171,6 +2592,14 @@ where (false, EnforcementMode::Audit, _) => "audit", (false, EnforcementMode::Enforce, _) => "deny", }; + correlation.record_policy_decision(if allowed { "allow" } else { "deny" }); + correlation.record_enforcement_action(if allowed { + "allow" + } else if config.enforcement == EnforcementMode::Audit { + "audit" + } else { + "block" + }); // Log every L7 decision as an OCSF HTTP Activity event. // Uses redacted_target (path only, no query params) to avoid logging secrets. @@ -2204,13 +2633,13 @@ where request_info.action, ctx.host, ctx.port, redacted_target, reason, )) .build(); - egress_emit!(event); + correlation.emit(event); } if allowed || config.enforcement == EnforcementMode::Audit { let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; let response_chain = chain.clone(); - let request_id = uuid::Uuid::new_v4().to_string(); + let request_id = correlation.request_id(); let websocket_chain = websocket_request.then(|| chain.clone()); // REST and websocket-upgrade policy evaluates only the method, // path, and query, which a middleware result cannot mutate, so no @@ -2223,12 +2652,20 @@ where engine.middleware_runner(), engine.generation_guard(), openshell_supervisor_middleware::TransformedBodyPolicy::NotPolicyRelevant, - &request_id, + request_id, ) .await; let req = match middleware_result? { MiddlewareApplyResult::Allowed(request) => request, MiddlewareApplyResult::Denied { denial, .. } => { + correlation.record_enforcement_action("block"); + correlation.record_response_status(403); + correlation.record_final_http_outcome(if denial.is_some() { + "denied" + } else { + "platform_error" + }); + correlation.record_error_stage("request_middleware"); let denied_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -2242,11 +2679,16 @@ where ctx, denial.as_ref(), &redacted_target, + Some(correlation.request_id()), ) .await?; return Ok(()); } MiddlewareApplyResult::AdmissionExhausted => { + correlation.record_enforcement_action("block"); + correlation.record_response_status(503); + correlation.record_final_http_outcome("platform_error"); + correlation.record_error_stage("middleware_admission"); let unavailable_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -2259,6 +2701,7 @@ where client, ctx, &redacted_target, + Some(correlation.request_id()), ) .await?; return Ok(()); @@ -2277,18 +2720,31 @@ where Ok(preflight) => preflight, Err(error) => { warn!(error = %error, "WebSocket middleware preflight failed"); - write_bad_gateway_response(client, None).await?; + correlation.record_enforcement_action("block"); + correlation.record_response_status(502); + correlation.record_final_http_outcome("platform_error"); + correlation.record_error_stage("websocket_middleware"); + write_bad_gateway_response(client, Some(correlation.request_id())).await?; return Ok(()); } }; crate::l7::middleware::emit_websocket_preflight_events(ctx, &preflight); if preflight.terminal_reason.is_some() { + correlation.record_enforcement_action("block"); + correlation.record_response_status(403); + correlation.record_final_http_outcome(if preflight.denial.is_some() { + "denied" + } else { + "platform_error" + }); + correlation.record_error_stage("websocket_middleware"); crate::l7::middleware::send_middleware_rejection_response( &req, client, ctx, preflight.denial.as_ref(), &redacted_target, + Some(correlation.request_id()), ) .await?; return Ok(()); @@ -2308,7 +2764,8 @@ where config, engine, request_info: &request_info, - request_id: &request_id, + request_id, + correlation: Some(&correlation), response_chain: &response_chain, websocket_middleware: middleware_session.is_some(), observation_context: observation_context.as_ref(), @@ -2382,6 +2839,8 @@ where } } else { // Enforce mode: deny with 403 and close connection (use redacted target) + correlation.record_response_status(403); + correlation.record_final_http_outcome("denied"); provider .deny_with_redacted_target( &req, @@ -2389,7 +2848,10 @@ where &reason, client, Some(&redacted_target), - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some( + crate::l7::rest::DenyResponseContext::from_l7_context(ctx) + .with_request_id(correlation.request_id()), + ), ) .await?; return Ok(()); @@ -2524,6 +2986,9 @@ where let req = parsed.request; let jsonrpc_info = parsed.info; + let correlation = RequestCorrelation::new(&req, ctx); + let mut audited_client = correlation.observe_response(client); + let client = &mut audited_client; let observer = EndpointObserver::begin_captured( ctx.endpoint_observation_tx.as_ref(), config, @@ -2535,11 +3000,18 @@ where if let Some(observer) = observer.as_ref() { observer.observe(EndpointResult::PolicyDenied); } - reject_request_authority_mismatch(client, ctx, &req.action).await?; + reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)).await?; return Ok(()); } - if deny_unsupported_upgrade_if_requested(&req, config, ctx, observer.as_ref(), client) - .await? + if deny_unsupported_upgrade_if_requested( + &req, + config, + ctx, + observer.as_ref(), + client, + Some(&correlation), + ) + .await? { return Ok(()); } @@ -2553,12 +3025,13 @@ where if let Some(observer) = observer.as_ref() { observer.observe_credential_failure(error.is_endpoint_mismatch()); } - reject_credential_resolution(client, ctx, &req.action, &error).await?; + reject_credential_resolution(client, ctx, &req.action, &error, Some(&correlation)) + .await?; return Ok(()); } }; - let Some(jsonrpc_info) = enforce_mcp_protocol_version( + let Some(jsonrpc_info) = enforce_mcp_protocol_version_correlated( config, &req, jsonrpc_info, @@ -2566,6 +3039,7 @@ where ctx, &redacted_target, observer.as_ref(), + Some(&correlation), ) .await? else { @@ -2585,8 +3059,9 @@ where let (allowed, reason, jsonrpc_log_info) = if let Some(reason) = hard_deny_reason { (false, reason, jsonrpc_info.clone()) } else { - let evaluation = - evaluate_jsonrpc_l7_request_for_log(engine, ctx, &request_info, &jsonrpc_info)?; + let evaluation = correlation.evaluate_policy(|| { + evaluate_jsonrpc_l7_request_for_log(engine, ctx, &request_info, &jsonrpc_info) + })?; (evaluation.allowed, evaluation.reason, evaluation.log_info) }; @@ -2600,6 +3075,14 @@ where (false, EnforcementMode::Audit) => "audit", (false, EnforcementMode::Enforce) => "deny", }; + correlation.record_policy_decision(if allowed { "allow" } else { "deny" }); + correlation.record_enforcement_action(if allowed { + "allow" + } else if config.enforcement == EnforcementMode::Audit && !force_deny { + "audit" + } else { + "block" + }); { let (action_id, disposition_id, severity) = match decision_str { @@ -2632,13 +3115,13 @@ where &reason, )) .build(); - egress_emit!(event); + correlation.emit(event); } if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; let response_chain = chain.clone(); - let request_id = uuid::Uuid::new_v4().to_string(); + let request_id = correlation.request_id(); // Policy admitted the original body above; re-check the body // against the same body-aware policy after every transforming // stage so a middleware cannot smuggle a denied operation to the @@ -2652,7 +3135,7 @@ where engine.middleware_runner(), engine.generation_guard(), openshell_supervisor_middleware::TransformedBodyPolicy::Reevaluate(&validate), - &request_id, + request_id, ) .await? { @@ -2661,6 +3144,14 @@ where if let Some(observer) = observer.as_ref() { observer.observe(EndpointResult::PolicyDenied); } + correlation.record_enforcement_action("block"); + correlation.record_response_status(403); + correlation.record_final_http_outcome(if denial.is_some() { + "denied" + } else { + "platform_error" + }); + correlation.record_error_stage("request_middleware"); let denied_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -2674,11 +3165,16 @@ where ctx, denial.as_ref(), &redacted_target, + Some(correlation.request_id()), ) .await?; return Ok(()); } MiddlewareApplyResult::AdmissionExhausted => { + correlation.record_enforcement_action("block"); + correlation.record_response_status(503); + correlation.record_final_http_outcome("platform_error"); + correlation.record_error_stage("middleware_admission"); let unavailable_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -2691,6 +3187,7 @@ where client, ctx, &redacted_target, + Some(correlation.request_id()), ) .await?; return Ok(()); @@ -2706,7 +3203,8 @@ where config, engine, request_info: &request_info, - request_id: &request_id, + request_id, + correlation: Some(&correlation), response_chain: &response_chain, websocket_middleware: false, observation_context: observation_context.as_ref(), @@ -2734,6 +3232,8 @@ where if let Some(observer) = observer.as_ref() { observer.observe(EndpointResult::PolicyDenied); } + correlation.record_response_status(403); + correlation.record_final_http_outcome("denied"); crate::l7::rest::RestProvider::default() .deny_with_redacted_target( &req, @@ -2741,7 +3241,10 @@ where &reason, client, Some(&redacted_target), - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some( + crate::l7::rest::DenyResponseContext::from_l7_context(ctx) + .with_request_id(correlation.request_id()), + ), ) .await?; return Ok(()); @@ -2798,12 +3301,24 @@ where return Ok(()); } }; + let correlation = RequestCorrelation::new(&req, ctx); + let mut audited_client = correlation.observe_response(client); + let client = &mut audited_client; if !request_authority_matches_endpoint(&req, ctx) { - reject_request_authority_mismatch(client, ctx, &req.action).await?; + reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)).await?; return Ok(()); } - if deny_unsupported_upgrade_if_requested(&req, config, ctx, None, client).await? { + if deny_unsupported_upgrade_if_requested( + &req, + config, + ctx, + None, + client, + Some(&correlation), + ) + .await? + { return Ok(()); } @@ -2826,7 +3341,11 @@ where } else { let detail = parse_rejection_detail(&e.to_string(), ParseRejectionMode::L7Endpoint); - emit_parse_rejection(ctx, &detail, "l7-graphql"); + correlation.record_policy_decision("deny"); + correlation.record_enforcement_action("block"); + correlation.record_final_http_outcome("denied"); + correlation.record_error_stage("request_body"); + emit_parse_rejection_correlated(ctx, &detail, "l7-graphql", Some(&correlation)); } return Ok(()); } @@ -2836,7 +3355,7 @@ where // without Host must still be denied if that body contains reserved // credential markers, so repeat the authority check on the full request. if !request_authority_matches_endpoint(&req, ctx) { - reject_request_authority_mismatch(client, ctx, &req.action).await?; + reject_request_authority_mismatch(client, ctx, &req.action, Some(&correlation)).await?; return Ok(()); } @@ -2847,7 +3366,8 @@ where let redacted_target = match secrets::redact_target_for_policy(&req.target) { Ok(target) => target, Err(error) => { - reject_credential_resolution(client, ctx, &req.action, &error).await?; + reject_credential_resolution(client, ctx, &req.action, &error, Some(&correlation)) + .await?; return Ok(()); } }; @@ -2869,7 +3389,7 @@ where let (allowed, reason) = if let Some(reason) = hard_deny_reason { (false, reason) } else { - evaluate_l7_request(engine, ctx, &request_info)? + correlation.evaluate_policy(|| evaluate_l7_request(engine, ctx, &request_info))? }; if close_if_stale(engine.generation_guard(), ctx) { @@ -2882,6 +3402,14 @@ where (false, EnforcementMode::Audit) => "audit", (false, EnforcementMode::Enforce) => "deny", }; + correlation.record_policy_decision(if allowed { "allow" } else { "deny" }); + correlation.record_enforcement_action(if allowed { + "allow" + } else if config.enforcement == EnforcementMode::Audit && !force_deny { + "audit" + } else { + "block" + }); { let (action_id, disposition_id, severity) = match decision_str { @@ -2914,13 +3442,13 @@ where request_info.action, ctx.host, ctx.port, redacted_target, reason, )) .build(); - egress_emit!(event); + correlation.emit(event); } if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { let chain = engine.query_middleware_chain(&middleware_network_input(ctx))?; let response_chain = chain.clone(); - let request_id = uuid::Uuid::new_v4().to_string(); + let request_id = correlation.request_id(); // Policy admitted the original body above; re-check the body // against the same body-aware policy after every transforming // stage so a middleware cannot smuggle a denied operation to the @@ -2934,12 +3462,20 @@ where engine.middleware_runner(), engine.generation_guard(), openshell_supervisor_middleware::TransformedBodyPolicy::Reevaluate(&validate), - &request_id, + request_id, ) .await? { MiddlewareApplyResult::Allowed(request) => request, MiddlewareApplyResult::Denied { denial, .. } => { + correlation.record_enforcement_action("block"); + correlation.record_response_status(403); + correlation.record_final_http_outcome(if denial.is_some() { + "denied" + } else { + "platform_error" + }); + correlation.record_error_stage("request_middleware"); let denied_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -2953,11 +3489,16 @@ where ctx, denial.as_ref(), &redacted_target, + Some(correlation.request_id()), ) .await?; return Ok(()); } MiddlewareApplyResult::AdmissionExhausted => { + correlation.record_enforcement_action("block"); + correlation.record_response_status(503); + correlation.record_final_http_outcome("platform_error"); + correlation.record_error_stage("middleware_admission"); let unavailable_request = crate::l7::provider::L7Request { action: request_info.action.clone(), target: redacted_target.clone(), @@ -2970,6 +3511,7 @@ where client, ctx, &redacted_target, + Some(correlation.request_id()), ) .await?; return Ok(()); @@ -2985,7 +3527,8 @@ where config, engine, request_info: &request_info, - request_id: &request_id, + request_id, + correlation: Some(&correlation), response_chain: &response_chain, websocket_middleware: false, observation_context: observation_context.as_ref(), @@ -3022,6 +3565,8 @@ where } } } else { + correlation.record_response_status(403); + correlation.record_final_http_outcome("denied"); crate::l7::rest::RestProvider::default() .deny_with_redacted_target( &req, @@ -3029,7 +3574,10 @@ where &reason, client, Some(&redacted_target), - Some(crate::l7::rest::DenyResponseContext::from_l7_context(ctx)), + Some( + crate::l7::rest::DenyResponseContext::from_l7_context(ctx) + .with_request_id(correlation.request_id()), + ), ) .await?; return Ok(()); @@ -3581,7 +4129,7 @@ where } }; if !request_authority_matches_endpoint(&req, ctx) { - reject_request_authority_mismatch(client, ctx, &req.action).await?; + reject_request_authority_mismatch(client, ctx, &req.action, None).await?; return Ok(()); } if close_if_stale(generation_guard, ctx) { @@ -3594,7 +4142,7 @@ where let redacted_target = match secrets::redact_target_for_policy(&req.target) { Ok(target) => target, Err(error) => { - reject_credential_resolution(client, ctx, &req.action, &error).await?; + reject_credential_resolution(client, ctx, &req.action, &error, None).await?; return Ok(()); } }; @@ -3663,6 +4211,7 @@ where ctx, denial.as_ref(), &redacted_target, + None, ) .await?; return Ok(()); @@ -3680,6 +4229,7 @@ where client, ctx, &redacted_target, + None, ) .await?; return Ok(()); @@ -3804,12 +4354,12 @@ mod trace_tests { ..Default::default() }; let mut response = Vec::new(); - // The in-memory span processor exports with a futures executor. tokio::task::block_in_place(|| { tokio::runtime::Handle::current().block_on(reject_request_authority_mismatch( &mut response, &ctx, "POST", + None, )) }) .unwrap(); @@ -3890,6 +4440,7 @@ mod tests { #[tokio::test] async fn body_denial_returns_actionable_local_json() { + const REQUEST_ID: &str = "8b02c179-9e6b-4eb0-948b-d9f40582fc44"; let req = crate::l7::provider::L7Request { action: "POST".into(), target: "/v1/responses".into(), query_params: TestHashMap::new(), raw_header: b"POST /v1/responses HTTP/1.1\r\nHost: api.openai.com\r\nContent-Length: 25\r\n\r\nopenshell:resolve:env:KEY".to_vec(), @@ -3897,7 +4448,12 @@ mod tests { }; let (mut client, mut caller) = tokio::io::duplex(4096); let (mut upstream, mut server) = tokio::io::duplex(4096); - let outcome = relay_http_request_with_credential_rejection( + let ctx = L7EvalContext { + policy_name: "body-policy".into(), + ..Default::default() + }; + let correlation = RequestCorrelation::with_request_id(&req, &ctx, REQUEST_ID.into()); + let outcome = relay_http_request_with_credential_rejection_observed( &req, &mut client, &mut upstream, @@ -3905,8 +4461,12 @@ mod tests { deny_uninspected_credentials: true, ..Default::default() }, - &L7EvalContext::default(), + &ctx, None, + RelayHttpInstrumentation { + observer: None, + correlation: Some(&correlation), + }, ) .await .unwrap(); @@ -3915,6 +4475,10 @@ mod tests { let mut response = String::new(); caller.read_to_string(&mut response).await.unwrap(); assert!(response.starts_with("HTTP/1.1 403 Forbidden")); + assert!( + response.contains(&format!("x-openshell-request-id: {REQUEST_ID}\r\n")), + "{response}" + ); let body: serde_json::Value = serde_json::from_str(response.split_once("\r\n\r\n").unwrap().1).unwrap(); assert_eq!( @@ -3922,6 +4486,7 @@ mod tests { "credential_placeholder_in_request_body" ); assert_eq!(body["error"]["reason"], "classification_unavailable"); + assert_eq!(body["request_id"], REQUEST_ID); let mut sent = String::new(); server.read_to_string(&mut sent).await.unwrap(); assert!(!sent.contains("openshell:resolve:")); @@ -3995,7 +4560,10 @@ mod tests { }, &L7EvalContext::default(), None, - Some(&observer), + RelayHttpInstrumentation { + observer: Some(&observer), + correlation: None, + }, ) .await; if disconnect_client { @@ -4331,6 +4899,7 @@ mod tests { resolver: &SecretResolver, options: crate::l7::rest::RelayRequestOptions<'_>, ) { + const REQUEST_ID: &str = "1329fced-3bb8-4810-bc25-45a1282997ad"; let (mut client_peer, mut client) = tokio::io::duplex(8192); let (mut upstream, mut upstream_peer) = tokio::io::duplex(8192); let ctx = L7EvalContext { @@ -4341,8 +4910,10 @@ mod tests { secret_resolver: Some(Arc::new(resolver.clone())), ..Default::default() }; + let correlation = + RequestCorrelation::with_request_id(&request, &ctx, REQUEST_ID.to_string()); - let outcome = relay_http_request_with_credential_rejection( + let outcome = relay_http_request_with_credential_rejection_observed( &request, &mut client, &mut upstream, @@ -4352,6 +4923,10 @@ mod tests { }, &ctx, None, + RelayHttpInstrumentation { + observer: None, + correlation: Some(&correlation), + }, ) .await .expect("typed credential denial"); @@ -4366,6 +4941,13 @@ mod tests { response.contains("credential_endpoint_mismatch"), "{response}" ); + assert!( + response.contains(&format!("x-openshell-request-id: {REQUEST_ID}\r\n")), + "{response}" + ); + let body: serde_json::Value = + serde_json::from_str(response.split_once("\r\n\r\n").unwrap().1).unwrap(); + assert_eq!(body["request_id"], REQUEST_ID); let mut forwarded = Vec::new(); upstream_peer.read_to_end(&mut forwarded).await.unwrap(); assert!( @@ -5020,6 +5602,70 @@ mod tests { .await; } + #[tokio::test] + async fn correlated_credential_unavailable_returns_local_receipt() { + const REQUEST_ID: &str = "250b9326-7a4f-4aa0-a6a9-7e0a1d96beaa"; + let request = crate::l7::provider::L7Request { + action: "GET".to_string(), + target: "/v1/resource".to_string(), + query_params: TestHashMap::new(), + raw_header: b"GET /v1/resource HTTP/1.1\r\nHost: api.example.test\r\n\r\n".to_vec(), + body_length: crate::l7::provider::BodyLength::None, + }; + let ctx = L7EvalContext { + host: "api.example.test".into(), + port: 443, + policy_name: "sigv4-policy".into(), + ..Default::default() + }; + let correlation = + RequestCorrelation::with_request_id(&request, &ctx, REQUEST_ID.to_string()); + let (mut client_peer, mut client) = tokio::io::duplex(8192); + let (mut upstream, mut upstream_peer) = tokio::io::duplex(8192); + + let outcome = relay_http_request_with_credential_rejection_observed( + &request, + &mut client, + &mut upstream, + crate::l7::rest::RelayRequestOptions { + credential_signing: crate::l7::CredentialSigning::SigV4NoBody, + signing_service: "execute-api", + signing_region: "us-west-2", + host: "api.example.test", + port: 443, + ..Default::default() + }, + &ctx, + None, + RelayHttpInstrumentation { + observer: None, + correlation: Some(&correlation), + }, + ) + .await + .expect("credential availability failure should be handled locally"); + assert!(outcome.is_none()); + drop(client); + drop(upstream); + + let mut response = String::new(); + client_peer.read_to_string(&mut response).await.unwrap(); + assert!(response.contains("500 Internal Server Error"), "{response}"); + assert!( + response.contains(&format!("x-openshell-request-id: {REQUEST_ID}\r\n")), + "{response}" + ); + let body: serde_json::Value = + serde_json::from_str(response.split_once("\r\n\r\n").unwrap().1).unwrap(); + assert_eq!(body["error"], "credential_unavailable"); + assert_eq!(body["request_id"], REQUEST_ID); + assert!(!response.contains("AWS credentials"), "{response}"); + + let mut forwarded = Vec::new(); + upstream_peer.read_to_end(&mut forwarded).await.unwrap(); + assert!(forwarded.is_empty()); + } + fn install_builtin_middleware(engine: &OpaEngine) { engine.set_middleware_runner_for_tests(openshell_supervisor_middleware::ChainRunner::new( openshell_supervisor_middleware_builtins::services() @@ -8776,6 +9422,124 @@ network_policies: .unwrap(); } + async fn assert_websocket_endpoint_rejects_plain_http(route_selected: bool) { + let data = r" +network_policies: + ws_api: + name: ws_api + endpoints: + - host: gateway.example.test + port: 443 + path: /ws + protocol: websocket + enforcement: enforce + rules: + - allow: + method: GET + path: /ws + binaries: + - { path: /usr/bin/node } +"; + let engine = OpaEngine::from_strings(TEST_POLICY, data).unwrap(); + let input = NetworkInput { + host: "gateway.example.test".into(), + port: 443, + binary_path: PathBuf::from("/usr/bin/node"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + let (endpoints, generation) = engine + .query_endpoint_configs_with_generation(&input) + .unwrap(); + let configs: Vec<_> = endpoints + .iter() + .map(|endpoint| crate::l7::parse_l7_config(endpoint).unwrap()) + .collect(); + let tunnel_engine = engine.clone_engine_for_tunnel(generation).unwrap(); + let ctx = L7EvalContext { + host: "gateway.example.test".into(), + port: 443, + request_default_port: Some(443), + policy_name: "ws_api".into(), + binary_path: "/usr/bin/node".into(), + ..Default::default() + }; + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let relay = tokio::spawn(async move { + if route_selected { + relay_with_route_selection( + &configs, + tunnel_engine, + &mut relay_client, + &mut relay_upstream, + &ctx, + ) + .await + } else { + relay_with_inspection( + &configs[0], + tunnel_engine, + &mut relay_client, + &mut relay_upstream, + &ctx, + ) + .await + } + }); + + app.write_all( + b"GET /ws HTTP/1.1\r\nHost: gateway.example.test\r\nConnection: close\r\n\r\n", + ) + .await + .unwrap(); + let response = tokio::time::timeout( + std::time::Duration::from_secs(1), + read_http_headers(&mut app), + ) + .await + .expect("denial should reach client"); + let response = String::from_utf8_lossy(&response); + assert!(response.contains("403 Forbidden"), "{response}"); + let request_id = response + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("x-openshell-request-id") + .then(|| value.trim()) + }) + .expect("denial should carry a supervisor receipt"); + assert!(uuid::Uuid::parse_str(request_id).is_ok()); + + let mut upstream_bytes = [0u8; 16]; + assert!(matches!( + tokio::time::timeout( + std::time::Duration::from_millis(100), + upstream.read(&mut upstream_bytes) + ) + .await, + Err(_) | Ok(Ok(0)) + )); + + drop(app); + tokio::time::timeout(std::time::Duration::from_secs(1), relay) + .await + .expect("relay should finish") + .unwrap() + .unwrap(); + } + + #[tokio::test] + async fn websocket_endpoint_plain_http_denial_has_receipt() { + assert_websocket_endpoint_rejects_plain_http(false).await; + } + + #[tokio::test] + async fn route_selected_websocket_plain_http_denial_has_receipt() { + assert_websocket_endpoint_rejects_plain_http(true).await; + } + #[tokio::test] async fn passthrough_relay_injects_token_grant_authorization_header() { let (generation_guard, ctx, fixture) = @@ -9839,12 +10603,23 @@ network_policies: .await .unwrap(); - let mut response = [0u8; 1024]; - let n = tokio::time::timeout(std::time::Duration::from_secs(1), app.read(&mut response)) - .await - .expect("denial should reach client") - .unwrap(); - assert!(String::from_utf8_lossy(&response[..n]).contains("403 Forbidden")); + let response = tokio::time::timeout( + std::time::Duration::from_secs(1), + read_http_headers(&mut app), + ) + .await + .expect("denial should reach client"); + let response = String::from_utf8_lossy(&response); + assert!(response.contains("403 Forbidden")); + let request_id = response + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("x-openshell-request-id") + .then(|| value.trim()) + }) + .expect("denial should carry a supervisor receipt"); + assert!(uuid::Uuid::parse_str(request_id).is_ok()); let mut upstream_bytes = [0u8; 16]; assert!(matches!( @@ -13290,6 +14065,7 @@ network_policies: #[tokio::test] async fn final_mcp_version_check_reclassifies_a_rewritten_initialize_body() { + const REQUEST_ID: &str = "36a15952-692c-44d8-a373-eeca9735fb6b"; let (config, _, ctx) = mcp_test_relay_context(); let final_body = br#"{"jsonrpc":"2.0","id":2,"method":"tools/list"}"#; let mut raw_header = format!( @@ -13306,14 +14082,17 @@ network_policies: body_length: crate::l7::provider::BodyLength::ContentLength(final_body.len() as u64), }; let (mut client, mut relay_client) = tokio::io::duplex(2048); + let correlation = + RequestCorrelation::with_request_id(&request, &ctx, REQUEST_ID.to_string()); - let allowed = enforce_final_mcp_protocol_version( + let allowed = enforce_final_mcp_protocol_version_correlated( &config, &request, &mut relay_client, &ctx, "/mcp", None, + Some(&correlation), ) .await .expect("final request inspection"); @@ -13321,6 +14100,7 @@ network_policies: !allowed, "rewritten non-initialize request must require a version" ); + drop(correlation); drop(relay_client); let mut response = Vec::new(); @@ -13331,6 +14111,13 @@ network_policies: response.contains("mcp_protocol_version_not_allowed"), "{response}" ); + assert!( + response.contains(&format!("x-openshell-request-id: {REQUEST_ID}\r\n")), + "{response}" + ); + let response_body: serde_json::Value = + serde_json::from_str(response.split_once("\r\n\r\n").unwrap().1).unwrap(); + assert_eq!(response_body["request_id"], REQUEST_ID); } #[tokio::test] diff --git a/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs b/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs new file mode 100644 index 0000000000..2f0b115680 --- /dev/null +++ b/crates/openshell-supervisor-network/src/l7/relay/correlation_dispatch_tests.rs @@ -0,0 +1,488 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Correlation contracts for the production single-config relay dispatch. + +use super::L7EvalContext; +use crate::l7::L7EndpointConfig; +use crate::opa::{NetworkInput, OpaEngine, TunnelPolicyEngine}; +use opentelemetry_sdk::trace::{InMemorySpanExporter, InMemorySpanExporterBuilder}; +use std::collections::BTreeMap; +use std::path::PathBuf; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; +use tracing::instrument::WithSubscriber as _; +use tracing_subscriber::layer::SubscriberExt as _; + +const POLICY: &str = include_str!("../../../data/sandbox-policy.rego"); + +struct Fixture { + config: L7EndpointConfig, + tunnel_engine: TunnelPolicyEngine, + middleware_engine: OpaEngine, + ctx: L7EvalContext, +} + +struct Exchange { + response: Vec, + forwarded: Vec, + span_attributes: BTreeMap, + ocsf_events: Vec, +} + +fn fixture(data: &str, host: &str, policy_name: &str) -> Fixture { + let middleware_engine = OpaEngine::from_strings(POLICY, data).expect("valid test policy"); + let input = NetworkInput { + host: host.into(), + port: 8000, + binary_path: PathBuf::from("/usr/bin/python3"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + let (endpoint_config, generation) = middleware_engine + .query_endpoint_config_with_generation(&input) + .expect("query endpoint config"); + let config = crate::l7::parse_l7_config( + &endpoint_config.expect("single configured endpoint should match"), + ) + .expect("parse endpoint config"); + let tunnel_engine = middleware_engine + .clone_engine_for_tunnel(generation) + .expect("pin test policy generation"); + let ctx = L7EvalContext { + host: host.into(), + port: 8000, + request_default_port: Some(8000), + policy_name: policy_name.into(), + binary_path: "/usr/bin/python3".into(), + ..Default::default() + }; + Fixture { + config, + tunnel_engine, + middleware_engine, + ctx, + } +} + +async fn read_http_message(reader: &mut R) -> Vec { + let mut message = Vec::new(); + let mut byte = [0_u8; 1]; + while !message.ends_with(b"\r\n\r\n") { + reader + .read_exact(&mut byte) + .await + .expect("complete HTTP headers"); + message.push(byte[0]); + } + let headers = std::str::from_utf8(&message).expect("ASCII test headers"); + let body_len = headers + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length").then(|| { + value + .trim() + .parse::() + .expect("numeric content length") + }) + }) + .unwrap_or_default(); + let header_len = message.len(); + message.resize(header_len + body_len, 0); + reader + .read_exact(&mut message[header_len..]) + .await + .expect("complete HTTP body"); + message +} + +fn request_id(response: &[u8]) -> &str { + let headers_end = response + .windows(4) + .position(|window| window == b"\r\n\r\n") + .expect("complete response headers"); + std::str::from_utf8(&response[..headers_end]) + .expect("ASCII response headers") + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("x-openshell-request-id") + .then(|| value.trim()) + }) + .expect("response carries a request receipt") +} + +fn enforcement_span_attributes(exporter: &InMemorySpanExporter) -> BTreeMap { + exporter + .get_finished_spans() + .expect("finished spans") + .into_iter() + .find(|span| span.name == "openshell.http.request") + .expect("HTTP enforcement span") + .attributes + .into_iter() + .map(|attribute| { + ( + attribute.key.as_str().to_string(), + attribute.value.to_string(), + ) + }) + .collect() +} + +async fn run_allowed( + fixture: Fixture, + request: Vec, + upstream_response: &'static [u8], +) -> Exchange { + let exporter = InMemorySpanExporterBuilder::new().build(); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let ocsf = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(openshell_otel::layer( + &provider, + "single-config-correlation-test", + )); + let subscriber = subscriber.with(ocsf.clone()); + let (mut app, mut relay_client) = tokio::io::duplex(32 * 1024); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(32 * 1024); + let Fixture { + config, + tunnel_engine, + middleware_engine, + ctx, + } = fixture; + + let exchange = async { + tracing::callsite::rebuild_interest_cache(); + tokio::join!( + crate::proxy::relay_inspected_http_stream_for_test( + &mut relay_client, + &mut relay_upstream, + vec![config], + tunnel_engine, + &middleware_engine, + &ctx, + ), + async { + app.write_all(&request).await.expect("send request"); + app.flush().await.expect("flush request"); + read_http_message(&mut app).await + }, + async { + let forwarded = read_http_message(&mut upstream).await; + upstream + .write_all(upstream_response) + .await + .expect("send upstream response"); + upstream.flush().await.expect("flush upstream response"); + forwarded + }, + ) + } + .with_subscriber(subscriber); + let (relay, response, forwarded) = + tokio::time::timeout(std::time::Duration::from_secs(5), Box::pin(exchange)) + .await + .expect("single-config relay exchange completes"); + relay.expect("relay succeeds"); + provider.force_flush().expect("flush spans"); + Exchange { + response, + forwarded, + span_attributes: enforcement_span_attributes(&exporter), + ocsf_events: ocsf.json_events(), + } +} + +async fn run_denied(fixture: Fixture, request: Vec) -> Exchange { + let exporter = InMemorySpanExporterBuilder::new().build(); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let ocsf = openshell_ocsf::test_support::OcsfCapture::default(); + let subscriber = tracing_subscriber::registry().with(openshell_otel::layer( + &provider, + "single-config-correlation-test", + )); + let subscriber = subscriber.with(ocsf.clone()); + let (mut app, mut relay_client) = tokio::io::duplex(32 * 1024); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(32 * 1024); + let Fixture { + config, + tunnel_engine, + middleware_engine, + ctx, + } = fixture; + + let exchange = async { + tracing::callsite::rebuild_interest_cache(); + tokio::join!( + crate::proxy::relay_inspected_http_stream_for_test( + &mut relay_client, + &mut relay_upstream, + vec![config], + tunnel_engine, + &middleware_engine, + &ctx, + ), + async { + app.write_all(&request).await.expect("send request"); + app.flush().await.expect("flush request"); + read_http_message(&mut app).await + }, + async { + let mut forwarded = Vec::new(); + let _ = tokio::time::timeout( + std::time::Duration::from_millis(100), + upstream.read_to_end(&mut forwarded), + ) + .await; + forwarded + }, + ) + } + .with_subscriber(subscriber); + let (relay, response, forwarded) = + tokio::time::timeout(std::time::Duration::from_secs(5), Box::pin(exchange)) + .await + .expect("single-config denial completes"); + relay.expect("relay succeeds"); + provider.force_flush().expect("flush spans"); + Exchange { + response, + forwarded, + span_attributes: enforcement_span_attributes(&exporter), + ocsf_events: ocsf.json_events(), + } +} + +fn assert_receipt_matches_span(exchange: &Exchange) { + let receipt = request_id(&exchange.response); + assert!( + uuid::Uuid::parse_str(receipt).is_ok(), + "canonical receipt: {receipt}" + ); + assert_eq!(exchange.span_attributes["openshell.request.id"], receipt); +} + +fn assert_policy_event_matches_receipt(exchange: &Exchange, engine_type: &str) { + let receipt = request_id(&exchange.response); + let event = exchange + .ocsf_events + .iter() + .find(|event| event["firewall_rule"]["type"] == engine_type) + .unwrap_or_else(|| { + panic!( + "missing {engine_type} policy event: {:?}", + exchange.ocsf_events + ) + }); + assert_eq!(event["metadata"]["correlation_uid"], receipt); +} + +fn mcp_fixture() -> Fixture { + fixture( + r#" +network_policies: + mcp_api: + name: mcp_api + endpoints: + - host: mcp.example.test + port: 8000 + path: /mcp + protocol: mcp + enforcement: enforce + mcp: + versions: ["2025-11-25"] + rules: + - allow: + method: initialize + binaries: + - { path: /usr/bin/python3 } +"#, + "mcp.example.test", + "mcp_api", + ) +} + +fn request(host: &str, path: &str, extra_headers: &str, body: &str) -> Vec { + format!( + "POST {path} HTTP/1.1\r\nHost: {host}:8000\r\nContent-Type: application/json\r\n{extra_headers}Content-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len(), + ) + .into_bytes() +} + +#[tokio::test] +async fn production_single_mcp_allow_correlates_span_and_response_receipt() { + let body = r#"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"test","version":"1"}}}"#; + let exchange = Box::pin(run_allowed( + mcp_fixture(), + request("mcp.example.test", "/mcp", "", body), + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}", + )) + .await; + + assert!(exchange.forwarded.starts_with(b"POST /mcp HTTP/1.1\r\n")); + assert!(exchange.response.starts_with(b"HTTP/1.1 200 OK\r\n")); + assert_receipt_matches_span(&exchange); + assert_policy_event_matches_receipt(&exchange, "l7-mcp"); + assert_eq!( + exchange.span_attributes["openshell.policy.decision"], + "allow" + ); + assert_eq!( + exchange.span_attributes["openshell.enforcement.action"], + "allow" + ); + assert_eq!( + exchange.span_attributes["openshell.http.outcome"], + "allowed" + ); + assert_eq!(exchange.span_attributes["http.response.status_code"], "200"); + assert_eq!(exchange.span_attributes["openshell.target.reached"], "true"); +} + +#[tokio::test] +async fn production_single_mcp_invalid_version_correlates_denial_and_receipt() { + let body = r#"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"test","version":"1"}}}"#; + let exchange = Box::pin(run_denied( + mcp_fixture(), + request( + "mcp.example.test", + "/mcp", + "MCP-Protocol-Version: 1900-01-01\r\n", + body, + ), + )) + .await; + + assert!(exchange.forwarded.is_empty()); + assert!( + exchange + .response + .starts_with(b"HTTP/1.1 400 Bad Request\r\n") + ); + assert!( + String::from_utf8_lossy(&exchange.response).contains("unsupported_mcp_protocol_version") + ); + assert_receipt_matches_span(&exchange); + assert_policy_event_matches_receipt(&exchange, "l7-mcp"); + assert_eq!( + exchange.span_attributes["openshell.policy.decision"], + "deny" + ); + assert_eq!( + exchange.span_attributes["openshell.enforcement.action"], + "block" + ); + assert_eq!(exchange.span_attributes["openshell.http.outcome"], "denied"); + assert_eq!(exchange.span_attributes["http.response.status_code"], "400"); + assert!( + !exchange + .span_attributes + .contains_key("openshell.error.stage") + ); + assert_eq!( + exchange.span_attributes["openshell.target.reached"], + "false" + ); +} + +#[tokio::test] +async fn production_single_mcp_policy_deny_correlates_span_event_and_receipt() { + let body = r#"{"jsonrpc":"2.0","id":1,"method":"tools/list"}"#; + let exchange = Box::pin(run_denied( + mcp_fixture(), + request( + "mcp.example.test", + "/mcp", + "MCP-Protocol-Version: 2025-11-25\r\n", + body, + ), + )) + .await; + + assert!(exchange.forwarded.is_empty()); + assert!(exchange.response.starts_with(b"HTTP/1.1 403 Forbidden\r\n")); + assert_receipt_matches_span(&exchange); + assert_policy_event_matches_receipt(&exchange, "l7-mcp"); + assert!(exchange.ocsf_events.iter().any(|event| { + event["firewall_rule"]["type"] == "l7-mcp" + && event["message"] + .as_str() + .is_some_and(|message| message.starts_with("JSONRPC_L7_REQUEST decision=deny")) + })); + assert_eq!( + exchange.span_attributes["openshell.policy.decision"], + "deny" + ); + assert_eq!( + exchange.span_attributes["openshell.enforcement.action"], + "block" + ); + assert_eq!(exchange.span_attributes["openshell.http.outcome"], "denied"); + assert_eq!(exchange.span_attributes["http.response.status_code"], "403"); + assert_eq!( + exchange.span_attributes["openshell.target.reached"], + "false" + ); +} + +#[tokio::test] +async fn production_single_graphql_allow_correlates_span_and_response_receipt() { + let graphql = fixture( + r" +network_policies: + graphql_api: + name: graphql_api + endpoints: + - host: graphql.example.test + port: 8000 + path: /graphql + protocol: graphql + enforcement: enforce + rules: + - allow: + operation_type: query + fields: [viewer] + binaries: + - { path: /usr/bin/python3 } +", + "graphql.example.test", + "graphql_api", + ); + let body = r#"{"query":"query { viewer }"}"#; + let exchange = Box::pin(run_allowed( + graphql, + request("graphql.example.test", "/graphql", "", body), + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 22\r\nConnection: close\r\n\r\n{\"data\":{\"viewer\":{}}}", + )) + .await; + + assert!( + exchange + .forwarded + .starts_with(b"POST /graphql HTTP/1.1\r\n") + ); + assert!(exchange.response.starts_with(b"HTTP/1.1 200 OK\r\n")); + assert_receipt_matches_span(&exchange); + assert_policy_event_matches_receipt(&exchange, "l7-graphql"); + assert_eq!( + exchange.span_attributes["openshell.policy.decision"], + "allow" + ); + assert_eq!( + exchange.span_attributes["openshell.enforcement.action"], + "allow" + ); + assert_eq!( + exchange.span_attributes["openshell.http.outcome"], + "allowed" + ); + assert_eq!(exchange.span_attributes["http.response.status_code"], "200"); + assert_eq!(exchange.span_attributes["openshell.target.reached"], "true"); +} diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 97c71f09b3..b4644e44cf 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -21,6 +21,7 @@ use http_response::{ }; use crate::l7::EndpointObserver; +use crate::l7::correlation::RequestCorrelation; use crate::l7::http::{parse_chunk_size_line, validate_chunk_trailer_line}; use crate::l7::provider::{BodyLength, L7Provider, L7Request, RelayOutcome}; use crate::opa::PolicyGenerationGuard; @@ -157,6 +158,7 @@ pub(crate) struct DenyResponseContext<'a> { pub(crate) port: Option, pub(crate) binary: Option<&'a str>, pub(crate) agent_proposals_enabled: bool, + pub(crate) request_id: Option<&'a str>, } impl<'a> DenyResponseContext<'a> { @@ -167,8 +169,15 @@ impl<'a> DenyResponseContext<'a> { port: Some(ctx.port), binary: Some(&ctx.binary_path), agent_proposals_enabled: ctx.agent_proposals.enabled(), + request_id: None, } } + + #[must_use] + pub(crate) fn with_request_id(mut self, request_id: &'a str) -> Self { + self.request_id = Some(request_id); + self + } } impl RestProvider { @@ -301,6 +310,8 @@ async fn parse_http_request( )?; } + buf = strip_platform_request_id_from_head(&buf); + Ok(Some(L7Request { action: method, target: canonical.path, @@ -332,6 +343,8 @@ pub(crate) fn request_from_buffered_http( let body_length = parse_body_length(header_str)?; let (_, query_params) = parse_target_query(query_target)?; + let raw_header = strip_platform_request_id_from_head(&raw_header); + Ok(L7Request { action: action.into(), target: target.into(), @@ -812,12 +825,33 @@ pub(crate) struct CredentialGenerationGuard<'a> { #[derive(Debug)] pub(crate) struct CredentialUnavailableError { reason: &'static str, + kind: CredentialUnavailableKind, } impl CredentialUnavailableError { fn new(reason: &'static str) -> Self { - Self { reason } + Self { + reason, + kind: CredentialUnavailableKind::MaterialUnavailable, + } + } + + fn generation_changed(reason: &'static str) -> Self { + Self { + reason, + kind: CredentialUnavailableKind::GenerationChanged, + } } + + pub(crate) fn generation_is_stale(&self) -> bool { + self.kind == CredentialUnavailableKind::GenerationChanged + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CredentialUnavailableKind { + MaterialUnavailable, + GenerationChanged, } impl fmt::Display for CredentialUnavailableError { @@ -859,9 +893,11 @@ impl<'a> CredentialGenerationGuard<'a> { { Ok(()) } else { - Err(miette::Report::new(CredentialUnavailableError::new( - "provider credential generation changed before upstream write", - ))) + Err(miette::Report::new( + CredentialUnavailableError::generation_changed( + "provider credential generation changed before upstream write", + ), + )) } } } @@ -897,6 +933,31 @@ pub(crate) async fn relay_http_request_with_response_middleware_guarded_observed response_middleware: Option>, observer: Option<&EndpointObserver>, ) -> Result +where + C: AsyncRead + AsyncWrite + Unpin, + U: AsyncRead + AsyncWrite + Unpin, +{ + relay_http_request_with_response_middleware_guarded_observed_correlated( + req, + client, + upstream, + options, + response_middleware, + observer, + None, + ) + .await +} + +pub(crate) async fn relay_http_request_with_response_middleware_guarded_observed_correlated( + req: &L7Request, + client: &mut C, + upstream: &mut U, + options: RelayRequestOptions<'_>, + response_middleware: Option>, + observer: Option<&EndpointObserver>, + correlation: Option<&RequestCorrelation>, +) -> Result where C: AsyncRead + AsyncWrite + Unpin, U: AsyncRead + AsyncWrite + Unpin, @@ -961,13 +1022,14 @@ where raw_header, body_length: req.body_length, }; - if !crate::l7::relay::enforce_final_mcp_protocol_version( + if !crate::l7::relay::enforce_final_mcp_protocol_version_correlated( validation.config, &outgoing, client, validation.ctx, validation.redacted_target, observer, + correlation, ) .await? { @@ -1269,6 +1331,7 @@ where websocket: websocket_response, client_requested_upgrade, observer, + correlation, }, response_middleware, ) @@ -1471,11 +1534,17 @@ where if contains_reserved_credential_marker_bytes(&trailer) { return Err(BodyCredentialError::Trailer.into()); } - write_body_bytes(upstream, &trailer, options).await?; - write_body_bytes(upstream, b"\r\n", options).await?; if trailer.is_empty() { + write_body_bytes(upstream, b"\r\n", options).await?; return Ok(()); } + if !http_field_line_has_name( + &trailer, + openshell_core::token_grant_failure::REQUEST_ID_HEADER, + ) { + write_body_bytes(upstream, &trailer, options).await?; + write_body_bytes(upstream, b"\r\n", options).await?; + } } } @@ -2291,6 +2360,89 @@ fn strip_header(headers: &[u8], strip_name: &str) -> Result> { Ok(out.into_bytes()) } +fn strip_platform_request_id_from_head(headers: &[u8]) -> Vec { + let header_end = headers + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map_or(headers.len(), |position| position + 4); + let (header_block, buffered_suffix) = headers.split_at(header_end); + let protected = openshell_core::token_grant_failure::REQUEST_ID_HEADER.as_bytes(); + let mut out = Vec::with_capacity(headers.len()); + let mut position = 0usize; + + while position < header_block.len() { + let line_end = header_block[position..] + .windows(2) + .position(|window| window == b"\r\n") + .map_or(header_block.len(), |end| position + end); + let line = &header_block[position..line_end]; + if line.is_empty() { + out.extend_from_slice(b"\r\n"); + break; + } + let Some(colon) = line.iter().position(|byte| *byte == b':') else { + out.extend_from_slice(line); + out.extend_from_slice(b"\r\n"); + position = line_end.saturating_add(2); + continue; + }; + let name = trim_ascii_ows(&line[..colon]); + if name.eq_ignore_ascii_case(protected) { + position = line_end.saturating_add(2); + continue; + } + if name.eq_ignore_ascii_case(b"trailer") { + let values: Vec<_> = line[colon + 1..] + .split(|byte| *byte == b',') + .filter(|value| !trim_ascii_ows(value).eq_ignore_ascii_case(protected)) + .collect(); + if values.len() != line[colon + 1..].split(|byte| *byte == b',').count() { + if values.iter().all(|value| trim_ascii_ows(value).is_empty()) { + position = line_end.saturating_add(2); + continue; + } + out.extend_from_slice(&line[..=colon]); + for (index, value) in values.iter().enumerate() { + if index > 0 { + out.push(b','); + } + out.extend_from_slice(value); + } + out.extend_from_slice(b"\r\n"); + position = line_end.saturating_add(2); + continue; + } + } + out.extend_from_slice(line); + out.extend_from_slice(b"\r\n"); + position = line_end.saturating_add(2); + } + out.extend_from_slice(buffered_suffix); + out +} + +fn trim_ascii_ows(mut value: &[u8]) -> &[u8] { + while value + .first() + .is_some_and(|byte| matches!(byte, b' ' | b'\t')) + { + value = &value[1..]; + } + while value + .last() + .is_some_and(|byte| matches!(byte, b' ' | b'\t')) + { + value = &value[..value.len() - 1]; + } + value +} + +fn http_field_line_has_name(line: &[u8], expected: &str) -> bool { + line.iter() + .position(|byte| *byte == b':') + .is_some_and(|end| line[..end].eq_ignore_ascii_case(expected.as_bytes())) +} + pub(crate) fn connection_nominated_header_names(headers: &[u8]) -> Result> { validate_http_request_header_block(headers)?; let header_str = @@ -2813,7 +2965,15 @@ async fn send_deny_response( context: Option>, ) -> Result<()> { let body = deny_response_body(req, policy_name, reason, redacted_target, context); - send_forbidden_json(policy_name, body, client).await + send_json_response_internal( + policy_name, + body, + client, + "403 Forbidden", + None, + context.and_then(|context| context.request_id), + ) + .await } /// Send a middleware-specific 403 response without policy-advisor guidance. @@ -2829,7 +2989,15 @@ pub(crate) async fn send_middleware_deny_response( context: Option>, ) -> Result<()> { let body = middleware_deny_response_body(req, policy_name, denial, redacted_target, context); - send_forbidden_json(policy_name, body, client).await + send_json_response_internal( + policy_name, + body_with_request_id(body, context.and_then(|context| context.request_id)), + client, + "403 Forbidden", + None, + context.and_then(|context| context.request_id), + ) + .await } /// Send a platform-owned response for fail-closed middleware failures. @@ -2845,7 +3013,15 @@ pub(crate) async fn send_middleware_failure_response( context: Option>, ) -> Result<()> { let body = middleware_failure_response_body(req, policy_name, redacted_target, context); - send_forbidden_json(policy_name, body, client).await + send_json_response_internal( + policy_name, + body_with_request_id(body, context.and_then(|context| context.request_id)), + client, + "403 Forbidden", + None, + context.and_then(|context| context.request_id), + ) + .await } /// Send a platform-owned availability response when shared middleware work @@ -2858,15 +3034,25 @@ pub(crate) async fn send_middleware_unavailable_response( context: Option>, ) -> Result<()> { let body = middleware_failure_response_body(req, policy_name, redacted_target, context); - send_json_response(policy_name, body, client, "503 Service Unavailable").await + send_json_response_internal( + policy_name, + body_with_request_id(body, context.and_then(|context| context.request_id)), + client, + "503 Service Unavailable", + None, + context.and_then(|context| context.request_id), + ) + .await } -async fn send_forbidden_json( - policy_name: &str, - body: serde_json::Value, - client: &mut C, -) -> Result<()> { - send_json_response(policy_name, body, client, "403 Forbidden").await +fn body_with_request_id( + mut body: serde_json::Value, + request_id: Option<&str>, +) -> serde_json::Value { + if let (Some(request_id), serde_json::Value::Object(fields)) = (request_id, &mut body) { + fields.insert("request_id".to_string(), serde_json::json!(request_id)); + } + body } /// Send a platform-owned JSON response with an explicit HTTP status. @@ -2887,15 +3073,68 @@ pub(crate) async fn send_json_response_with_allow( client: &mut C, status: &str, allowed_methods: Option<&'static str>, +) -> Result<()> { + send_json_response_internal(policy_name, body, client, status, allowed_methods, None).await +} + +pub(crate) async fn send_json_response_with_allow_and_request_id( + policy_name: &str, + mut body: serde_json::Value, + client: &mut C, + status: &str, + allowed_methods: Option<&'static str>, + request_id: &str, +) -> Result<()> { + if let serde_json::Value::Object(fields) = &mut body { + fields.insert("request_id".to_string(), serde_json::json!(request_id)); + } + send_json_response_internal( + policy_name, + body, + client, + status, + allowed_methods, + Some(request_id), + ) + .await +} + +pub(crate) async fn send_json_response_with_request_id( + policy_name: &str, + mut body: serde_json::Value, + client: &mut C, + status: &str, + request_id: &str, +) -> Result<()> { + if let serde_json::Value::Object(fields) = &mut body { + fields.insert("request_id".to_string(), serde_json::json!(request_id)); + } + send_json_response_internal(policy_name, body, client, status, None, Some(request_id)).await +} + +async fn send_json_response_internal( + policy_name: &str, + body: serde_json::Value, + client: &mut C, + status: &str, + allowed_methods: Option<&'static str>, + request_id: Option<&str>, ) -> Result<()> { let body_bytes = body.to_string(); let allow = allowed_methods.map_or_else(String::new, |methods| format!("Allow: {methods}\r\n")); + let request_id = request_id.map_or_else(String::new, |request_id| { + format!( + "{}: {request_id}\r\n", + openshell_core::token_grant_failure::REQUEST_ID_HEADER + ) + }); let response = format!( "HTTP/1.1 {status}\r\n\ Content-Type: application/json\r\n\ Content-Length: {}\r\n\ X-OpenShell-Policy: {}\r\n\ {allow}\ + {request_id}\ Connection: close\r\n\ \r\n\ {}", @@ -3029,6 +3268,9 @@ fn deny_response_body( if let Some(binary) = binary { body.insert("binary".to_string(), serde_json::json!(binary)); } + if let Some(request_id) = context.request_id { + body.insert("request_id".to_string(), serde_json::json!(request_id)); + } body.insert( "rule_missing".to_string(), serde_json::Value::Object(rule_missing), @@ -3251,6 +3493,7 @@ where let mut trailer_count = 0usize; let mut trailer_bytes = 0usize; loop { + let pending_start = input.pending_len(); let trailer_line = input .read_line( reader, @@ -3276,6 +3519,12 @@ where ); return Ok(()); } + if http_field_line_has_name( + &trailer_line, + openshell_core::token_grant_failure::REQUEST_ID_HEADER, + ) { + input.pending.truncate(pending_start); + } if trailer_count == MAX_CHUNK_TRAILER_FIELDS { return Err(miette!( "Chunk trailers exceed {MAX_CHUNK_TRAILER_FIELDS} fields" @@ -4631,6 +4880,7 @@ mod tests { port: Some(443), binary: Some("/usr/bin/gh"), agent_proposals_enabled: true, + request_id: None, }), ); @@ -4695,6 +4945,7 @@ mod tests { port: Some(443), binary: Some("/usr/bin/gh"), agent_proposals_enabled: false, + request_id: None, }), ); @@ -4798,6 +5049,7 @@ mod tests { port: Some(443), binary: Some("/usr/bin/curl"), agent_proposals_enabled: true, + request_id: None, }), ); @@ -4836,6 +5088,7 @@ mod tests { port: Some(443), binary: Some("/usr/bin/gh"), agent_proposals_enabled: true, + request_id: Some("request-123"), }), ) .await @@ -4850,12 +5103,14 @@ mod tests { assert!(response.starts_with("HTTP/1.1 403 Forbidden")); assert!(response.contains("Content-Type: application/json")); assert!(response.contains("X-OpenShell-Policy: github-readonly")); + assert!(response.contains("x-openshell-request-id: request-123")); let (_, body) = response.split_once("\r\n\r\n").unwrap(); let body: serde_json::Value = serde_json::from_str(body).unwrap(); assert_eq!(body["error"], "policy_denied"); assert_eq!(body["method"], "POST"); assert_eq!(body["path"], "/user/repos"); + assert_eq!(body["request_id"], "request-123"); assert_eq!(body["rule_missing"]["host"], "api.github.com"); assert_eq!(body["next_steps"][2]["action"], "inspect_recent_denials"); assert!(body["agent_guidance"].as_str().unwrap().contains("retry")); @@ -5005,6 +5260,42 @@ mod tests { assert_eq!(forwarded, expected); } + #[tokio::test] + async fn relay_chunked_strips_platform_request_id_trailer() { + let mut reader = + b"3\r\nabc\r\n0\r\nX-OpenShell-Request-ID: forged\r\nX-Checksum: ok\r\n\r\n".as_slice(); + let mut writer = Vec::new(); + + relay_chunked(&mut reader, &mut writer, &[], None) + .await + .expect("chunked body should relay"); + + assert_eq!(writer, b"3\r\nabc\r\n0\r\nX-Checksum: ok\r\n\r\n"); + } + + #[tokio::test] + async fn guarded_chunked_request_strips_platform_request_id_trailer() { + let mut reader = + b"3\r\nabc\r\n0\r\nx-openshell-request-id: forged\r\nX-Checksum: ok\r\n\r\n".as_slice(); + let mut writer = Vec::new(); + + relay_chunked_with_marker_guard( + &mut reader, + &mut writer, + &[], + RelayRequestOptions::default(), + ) + .await + .expect("guarded chunked body should relay"); + + assert!( + !String::from_utf8_lossy(&writer) + .to_ascii_lowercase() + .contains("x-openshell-request-id") + ); + assert!(String::from_utf8_lossy(&writer).contains("X-Checksum: ok\r\n")); + } + #[tokio::test] async fn relay_chunked_handles_tiny_chunks_without_an_aggregate_framing_limit() { let mut chunked_body = Vec::new(); @@ -5262,6 +5553,18 @@ mod tests { assert!(parse_body_length(headers).is_err()); } + #[test] + fn platform_request_id_is_stripped_without_touching_body() { + let suffix = b"4\r\nbody\r\n0\r\nX-OpenShell-Request-ID: forged-trailer\r\n\r\n"; + let mut request = b"POST /api HTTP/1.1\r\nHost: x\r\nTransfer-Encoding: chunked\r\nX-Opaque: \xff\xfe\r\nX-OpenShell-Request-ID: forged-head\r\nTrailer: X-OpenShell-Request-ID, X-Checksum\r\n\r\n".to_vec(); + request.extend_from_slice(suffix); + + let stripped = strip_platform_request_id_from_head(&request); + let mut expected = b"POST /api HTTP/1.1\r\nHost: x\r\nTransfer-Encoding: chunked\r\nX-Opaque: \xff\xfe\r\nTrailer: X-Checksum\r\n\r\n".to_vec(); + expected.extend_from_slice(suffix); + assert_eq!(stripped, expected); + } + #[test] fn strip_connection_nominated_headers_before_forwarding() { let raw = b"GET /api HTTP/1.1\r\nHost: x\r\nX-Guard: hidden\r\nConnection: keep-alive, x-guard\r\nKeep-Alive: timeout=5\r\nX-Visible: yes\r\n\r\n"; @@ -6510,11 +6813,31 @@ mod tests { #[tokio::test] async fn endpoint_observation_rejects_malformed_http_status() { - assert_observed_response_result( - b"not-http 200 nope\r\nContent-Length: 0\r\n\r\n", - EndpointResult::TransportFailed, + let (observer, mut receiver) = test_endpoint_observer().await; + let mut upstream = &b"not-http 200 nope\r\nContent-Length: 0\r\n\r\n"[..]; + let mut client = Vec::new(); + + let result = relay_response( + "POST", + &mut upstream, + &mut client, + RelayResponseOptions { + observer: Some(&observer), + ..Default::default() + }, + None, ) .await; + + assert!(result.is_err(), "malformed response must fail closed"); + assert!(client.is_empty()); + assert!(matches!( + receiver.try_recv().expect("transport failure observation"), + EndpointStatusCommand::Observe { + result: EndpointResult::TransportFailed, + .. + } + )); } #[tokio::test] @@ -7605,8 +7928,16 @@ mod tests { #[tokio::test] async fn response_middleware_rejects_malformed_response_fields_before_commit() { + let (outcome, delivered) = run_response_middleware_relay( + b"HTTP/1.1 200 OK\r\nBad Name: value\r\nContent-Length: 0\r\n\r\n", + "GET", + ResponseRelayScript::WholeBody, + ) + .await; + assert!(outcome.is_err()); + assert!(delivered.is_empty()); + for response in [ - b"HTTP/1.1 200 OK\r\nBad Name: value\r\nContent-Length: 0\r\n\r\n".as_slice(), b"HTTP/1.1 200 OK\r\nTrailer: content-length\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n\r\n".as_slice(), b"HTTP/1.1 200 OK\r\nConnection: x-private\r\nTrailer: x-private\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n\r\n".as_slice(), ] { @@ -7856,17 +8187,30 @@ mod tests { #[tokio::test] async fn response_middleware_obs_text_does_not_bypass_unsafe_headers() { for response in [ - b"HTTP/1.1 200 OK\r\nX-Legacy: \xff\r\nBad Name: value\r\nContent-Length: 2\r\n\r\nok".as_slice(), + b"HTTP/1.1 200 OK\r\nX-Legacy: \xff\r\nBad Name: value\r\nContent-Length: 2\r\n\r\nok" + .as_slice(), b"HTTP/1.1 200 OK\r\nX-Legacy: \xff\x00\r\nContent-Length: 2\r\n\r\nok".as_slice(), - b"HTTP/1.1 200 OK\r\nX-Legacy: \xff\r\nTrailer: content-length\r\nContent-Length: 2\r\n\r\nok".as_slice(), ] { let (outcome, delivered) = run_response_middleware_relay_with_error( - response, "GET", ResponseRelayScript::HeadersOnly, + response, + "GET", + ResponseRelayScript::HeadersOnly, openshell_supervisor_middleware::OnError::FailOpen, - ).await; - assert!(matches!(outcome.unwrap(), RelayOutcome::Consumed)); - assert!(delivered.starts_with(b"HTTP/1.1 502 Bad Gateway\r\n")); + ) + .await; + assert!(outcome.is_err()); + assert!(delivered.is_empty()); } + + let (outcome, delivered) = run_response_middleware_relay_with_error( + b"HTTP/1.1 200 OK\r\nX-Legacy: \xff\r\nTrailer: content-length\r\nContent-Length: 2\r\n\r\nok", + "GET", + ResponseRelayScript::HeadersOnly, + openshell_supervisor_middleware::OnError::FailOpen, + ) + .await; + assert!(matches!(outcome.unwrap(), RelayOutcome::Consumed)); + assert!(delivered.starts_with(b"HTTP/1.1 502 Bad Gateway\r\n")); } #[tokio::test] diff --git a/crates/openshell-supervisor-network/src/l7/rest/http_response.rs b/crates/openshell-supervisor-network/src/l7/rest/http_response.rs index 0113dc4323..606767d3fc 100644 --- a/crates/openshell-supervisor-network/src/l7/rest/http_response.rs +++ b/crates/openshell-supervisor-network/src/l7/rest/http_response.rs @@ -4,6 +4,7 @@ //! HTTP response relay and pre-return middleware integration. use super::*; +use crate::l7::correlation::RequestCorrelation; /// Default wall-clock bound shared by whole-body stages in one response. pub const DEFAULT_HTTP_RESPONSE_WHOLE_BODY_TIMEOUT: std::time::Duration = @@ -26,6 +27,7 @@ pub(super) struct RelayResponseOptions<'a> { pub(super) client_requested_upgrade: bool, pub(super) websocket: Option, pub(super) observer: Option<&'a EndpointObserver>, + pub(super) correlation: Option<&'a RequestCorrelation>, } impl Default for RelayResponseOptions<'_> { @@ -35,6 +37,7 @@ impl Default for RelayResponseOptions<'_> { client_requested_upgrade: true, websocket: None, observer: None, + correlation: None, } } } @@ -57,7 +60,7 @@ where // Forward interim responses unchanged, but retain the final response head // until response middleware preflight completes. let mut informational_header_bytes = 0; - let header_end = loop { + let mut header_end = loop { let header_end = buf .windows(4) .position(|w| w == b"\r\n\r\n") @@ -73,16 +76,21 @@ where } if let Some(header_end) = header_end { + if let Err(error) = validate_http_response_header_block(&buf[..header_end]) { + if let Some(observer) = options.observer.as_ref() { + observer.observe(EndpointResult::TransportFailed); + } + record_upstream_response_error(options.correlation, "upstream_response_headers"); + return Err(error); + } let header_str = String::from_utf8_lossy(&buf[..header_end]); if matches!( parse_observed_http_status_code(&header_str), Some(100 | 102..=199) ) { informational_header_bytes += header_end; - client - .write_all(&buf[..header_end]) - .await - .into_diagnostic()?; + let sanitized = strip_platform_request_id_from_head(&buf[..header_end]); + client.write_all(&sanitized).await.into_diagnostic()?; client.flush().await.into_diagnostic()?; buf.drain(..header_end); continue; @@ -95,7 +103,9 @@ where if let Some(observer) = options.observer.as_ref() { observer.observe(EndpointResult::TransportFailed); } - if !buf.is_empty() { + if let Some(correlation) = options.correlation { + send_incomplete_response(client, correlation).await?; + } else if !buf.is_empty() { client.write_all(&buf).await.into_diagnostic()?; } return Ok(RelayOutcome::Consumed); @@ -103,9 +113,24 @@ where buf.extend_from_slice(&tmp[..n]); }; + let sanitized = strip_platform_request_id_from_head(&buf[..header_end]); + if sanitized.len() != header_end { + let overflow = buf.split_off(header_end); + buf = sanitized; + header_end = buf.len(); + buf.extend_from_slice(&overflow); + } + + if let Some(correlation) = options.correlation { + correlation.record_target_reached(); + } + // Parse response framing let header_str = String::from_utf8_lossy(&buf[..header_end]); let observed_status = parse_observed_http_status_code(&header_str); + if let (Some(correlation), Some(status_code)) = (options.correlation, observed_status) { + correlation.record_upstream_status(status_code); + } if let Some(observer) = options.observer.as_ref() { match observed_status { Some(status_code) if status_code < 400 => { @@ -145,10 +170,8 @@ where options.websocket_extensions, options.websocket.as_ref(), )?; - client - .write_all(&buf[..header_end]) - .await - .into_diagnostic()?; + let final_head = final_response_head(&buf[..header_end], options.correlation); + client.write_all(&final_head).await.into_diagnostic()?; client.flush().await.into_diagnostic()?; let overflow = buf[header_end..].to_vec(); debug!( @@ -156,6 +179,7 @@ where overflow_bytes = overflow.len(), "101 Switching Protocols — signaling protocol upgrade" ); + record_allowed_response(options.correlation); return Ok(RelayOutcome::Upgraded { overflow, websocket_permessage_deflate, @@ -175,6 +199,7 @@ where body_length, server_wants_close || http_10_closes_by_default, event_stream, + options.correlation, )) .await? { @@ -185,13 +210,15 @@ where }; } + let final_head = final_response_head(&buf[..header_end], options.correlation); + // Bodiless responses (HEAD, 1xx, 204, 304): forward headers only, skip body if is_bodiless_response(request_method, status_code) { - client - .write_all(&buf[..header_end]) - .await - .into_diagnostic()?; - return finish_response(client, server_wants_close || http_10_closes_by_default).await; + client.write_all(&final_head).await.into_diagnostic()?; + let outcome = + finish_response(client, server_wants_close || http_10_closes_by_default).await?; + record_allowed_response(options.correlation); + return Ok(outcome); } // No explicit framing (no Content-Length, no Transfer-Encoding). @@ -200,7 +227,7 @@ where if server_wants_close || http_10_closes_by_default || event_stream { // Server indicated it will close, or this is a streaming response // such as SSE where the body is intentionally delimited by EOF. - let before_end = &buf[..header_end - 2]; + let before_end = &final_head[..final_head.len() - 2]; client.write_all(before_end).await.into_diagnostic()?; if server_wants_close || http_10_closes_by_default { client @@ -222,25 +249,21 @@ where } client.flush().await.into_diagnostic()?; client.shutdown().await.into_diagnostic()?; + record_allowed_response(options.correlation); return Ok(RelayOutcome::Consumed); } // No Connection: close — an HTTP/1.1 keep-alive server that omits // framing headers has an empty body. Forward headers and continue // the relay loop instead of blocking on relay_until_eof. debug!("BodyLength::None without Connection: close — treating body as empty"); - client - .write_all(&buf[..header_end]) - .await - .into_diagnostic()?; + client.write_all(&final_head).await.into_diagnostic()?; client.flush().await.into_diagnostic()?; + record_allowed_response(options.correlation); return Ok(RelayOutcome::Reusable); } // Hold buffered chunk framing until relay_chunked validates it. - client - .write_all(&buf[..header_end]) - .await - .into_diagnostic()?; + client.write_all(&final_head).await.into_diagnostic()?; let overflow = &buf[header_end..]; let overflow_len = (buf.len() - header_end) as u64; @@ -267,7 +290,247 @@ where "relay_response complete (explicit framing)" ); - finish_response(client, server_wants_close || http_10_closes_by_default).await + let outcome = finish_response(client, server_wants_close || http_10_closes_by_default).await?; + record_allowed_response(options.correlation); + Ok(outcome) +} + +async fn send_incomplete_response( + client: &mut C, + correlation: &RequestCorrelation, +) -> Result<()> { + record_upstream_response_error(Some(correlation), "upstream_response_incomplete"); + let body = serde_json::json!({ + "error": "upstream_response_incomplete", + "request_id": correlation.request_id(), + }) + .to_string(); + let response = format!( + "HTTP/1.1 502 Bad Gateway\r\nContent-Type: application/json\r\n{}: {}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + openshell_core::token_grant_failure::REQUEST_ID_HEADER, + correlation.request_id(), + body.len(), + body, + ); + client + .write_all(response.as_bytes()) + .await + .into_diagnostic()?; + client.flush().await.into_diagnostic() +} + +fn record_allowed_response(correlation: Option<&RequestCorrelation>) { + if let Some(correlation) = correlation { + correlation.record_final_http_outcome("allowed"); + } +} + +fn record_denied_response(correlation: Option<&RequestCorrelation>, stage: &str) { + if let Some(correlation) = correlation { + correlation.record_response_status(403); + correlation.record_final_http_outcome("denied"); + correlation.record_error_stage(stage); + } +} + +fn record_platform_response_error(correlation: Option<&RequestCorrelation>, stage: &str) { + if let Some(correlation) = correlation { + correlation.record_response_status(502); + correlation.record_final_http_outcome("platform_error"); + correlation.record_error_stage(stage); + } +} + +fn record_upstream_response_error(correlation: Option<&RequestCorrelation>, stage: &str) { + if let Some(correlation) = correlation { + correlation.record_response_status(502); + correlation.record_final_http_outcome("upstream_error"); + correlation.record_error_stage(stage); + } +} + +fn record_committed_response_error( + correlation: Option<&RequestCorrelation>, + outcome: &'static str, + stage: &str, +) { + if let Some(correlation) = correlation { + correlation.record_final_http_outcome(outcome); + correlation.record_error_stage(stage); + } +} + +fn validate_http_response_header_block(headers: &[u8]) -> Result<()> { + if !headers.ends_with(b"\r\n\r\n") { + return Err(miette!("HTTP response header block is incomplete")); + } + + for (index, byte) in headers.iter().copied().enumerate() { + match byte { + b'\r' if headers.get(index + 1) != Some(&b'\n') => { + return Err(miette!("HTTP response headers contain a bare CR")); + } + b'\n' if index == 0 || headers.get(index - 1) != Some(&b'\r') => { + return Err(miette!("HTTP response headers contain a bare LF")); + } + _ => {} + } + } + + let header_lines = &headers[..headers.len() - 2]; + let mut lines = header_lines + .split(|byte| *byte == b'\n') + .map(|line| line.strip_suffix(b"\r").unwrap_or(line)); + let status_line = lines + .next() + .ok_or_else(|| miette!("HTTP response status line is missing"))?; + validate_http_response_status_line(status_line)?; + + while let Some(line) = lines.next() { + if line.is_empty() { + if lines.next().is_some() { + return Err(miette!( + "HTTP response header block contains an unexpected empty line" + )); + } + break; + } + if line + .first() + .is_some_and(|byte| matches!(byte, b' ' | b'\t')) + { + return Err(miette!( + "HTTP response header continuation lines are not supported" + )); + } + let colon = line + .iter() + .position(|byte| *byte == b':') + .ok_or_else(|| miette!("HTTP response header field is malformed"))?; + let name = &line[..colon]; + if name.last().is_some_and(|byte| matches!(byte, b' ' | b'\t')) { + return Err(miette!( + "HTTP response header field contains whitespace before the colon" + )); + } + http::HeaderName::from_bytes(name) + .map_err(|_| miette!("HTTP response field name is malformed"))?; + http::HeaderValue::from_bytes(&line[colon + 1..]) + .map_err(|_| miette!("HTTP response field value is malformed"))?; + } + Ok(()) +} + +fn validate_http_response_status_line(status_line: &[u8]) -> Result<()> { + let remainder = if let Some(remainder) = status_line.strip_prefix(b"HTTP/1.0 ") { + remainder + } else if let Some(remainder) = status_line.strip_prefix(b"HTTP/1.1 ") { + remainder + } else { + return Err(miette!( + "HTTP response status line must use HTTP/1.0 or HTTP/1.1" + )); + }; + if remainder.len() < 4 || remainder[3] != b' ' { + return Err(miette!("HTTP response status line is malformed")); + } + http::StatusCode::from_bytes(&remainder[..3]) + .map_err(|_| miette!("HTTP response status code is invalid"))?; + http::HeaderValue::from_bytes(&remainder[4..]) + .map_err(|_| miette!("HTTP response reason phrase is malformed"))?; + Ok(()) +} + +fn final_response_head(headers: &[u8], correlation: Option<&RequestCorrelation>) -> Vec { + let stripped = strip_platform_request_id_from_head(headers); + let stripped = strip_platform_request_id_from_connection(&stripped); + let Some(correlation) = correlation else { + return stripped; + }; + let Some(header_end) = stripped.windows(4).position(|window| window == b"\r\n\r\n") else { + return stripped; + }; + let receipt = format!( + "{}: {}\r\n\r\n", + openshell_core::token_grant_failure::REQUEST_ID_HEADER, + correlation.request_id(), + ); + let mut output = Vec::with_capacity(stripped.len() + receipt.len()); + output.extend_from_slice(&stripped[..header_end + 2]); + output.extend_from_slice(receipt.as_bytes()); + output.extend_from_slice(&stripped[header_end + 4..]); + output +} + +fn strip_platform_request_id_from_connection(headers: &[u8]) -> Vec { + let header_end = headers + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map_or(headers.len(), |position| position + 4); + let (header_block, buffered_suffix) = headers.split_at(header_end); + let protected = openshell_core::token_grant_failure::REQUEST_ID_HEADER.as_bytes(); + let mut output = Vec::with_capacity(headers.len()); + let mut position = 0usize; + + while position < header_block.len() { + let line_end = header_block[position..] + .windows(2) + .position(|window| window == b"\r\n") + .map_or(header_block.len(), |end| position + end); + let line = &header_block[position..line_end]; + if line.is_empty() { + output.extend_from_slice(b"\r\n"); + break; + } + + let Some(colon) = line.iter().position(|byte| *byte == b':') else { + output.extend_from_slice(line); + output.extend_from_slice(b"\r\n"); + position = line_end.saturating_add(2); + continue; + }; + let name = trim_ascii_ows(&line[..colon]); + if !name.eq_ignore_ascii_case(b"connection") { + output.extend_from_slice(line); + output.extend_from_slice(b"\r\n"); + position = line_end.saturating_add(2); + continue; + } + + let mut removed_protected = false; + let retained: Vec<_> = line[colon + 1..] + .split(|byte| *byte == b',') + .filter_map(|value| { + let token = trim_ascii_ows(value); + if token.eq_ignore_ascii_case(protected) { + removed_protected = true; + None + } else if token.is_empty() { + None + } else { + Some(token) + } + }) + .collect(); + if !removed_protected { + output.extend_from_slice(line); + output.extend_from_slice(b"\r\n"); + } else if !retained.is_empty() { + output.extend_from_slice(&line[..=colon]); + output.push(b' '); + for (index, token) in retained.iter().enumerate() { + if index > 0 { + output.extend_from_slice(b", "); + } + output.extend_from_slice(token); + } + output.extend_from_slice(b"\r\n"); + } + position = line_end.saturating_add(2); + } + + output.extend_from_slice(buffered_suffix); + output } /// Body framing determines when delivery finishes, not whether another @@ -299,6 +562,7 @@ async fn relay_response_through_middleware( body_length: BodyLength, server_wants_close: bool, event_stream: bool, + response_correlation: Option<&RequestCorrelation>, ) -> Result> where U: AsyncRead + Unpin, @@ -342,6 +606,8 @@ where request_method, middleware.policy_name, &middleware.target, + "response_middleware_headers", + response_correlation, ) .await?; return Ok(Some(RelayOutcome::Consumed)); @@ -387,6 +653,8 @@ where request_method, middleware.policy_name, &middleware.target, + "response_middleware_preflight", + response_correlation, ) .await?; return Ok(Some(RelayOutcome::Consumed)); @@ -436,6 +704,8 @@ where middleware.policy_name, &middleware.target, denial, + "response_middleware_preflight", + response_correlation, ) .await?; } else { @@ -444,6 +714,8 @@ where request_method, middleware.policy_name, &middleware.target, + "response_middleware_preflight", + response_correlation, ) .await?; } @@ -477,8 +749,10 @@ where body_length, server_wants_close, event_stream, + response_correlation, ) .await?; + record_allowed_response(response_correlation); return Ok(Some(outcome)); }; @@ -510,6 +784,8 @@ where request_method, middleware.policy_name, &middleware.target, + "response_middleware_finish", + response_correlation, ) .await?; return Ok(Some(RelayOutcome::Consumed)); @@ -540,8 +816,10 @@ where server_wants_close, &[], ); + let head = final_response_head(&head, response_correlation); client.write_all(&head).await.into_diagnostic()?; client.flush().await.into_diagnostic()?; + record_allowed_response(response_correlation); return Ok(Some(if server_wants_close { RelayOutcome::Consumed } else { @@ -574,6 +852,7 @@ where server_wants_close || close_delimited_output, downstream_trailers, ); + let streaming_head = final_response_head(&streaming_head, response_correlation); let mut committed = !whole_body; if committed { if let Err(error) = client.write_all(&streaming_head).await { @@ -652,6 +931,19 @@ where } crate::telemetry::emit_decision(events, None); if committed { + if middleware_stop.is_some_and(|stop| stop.failure.denial.is_some()) { + record_committed_response_error( + response_correlation, + "denied", + "response_middleware_body", + ); + } else { + record_committed_response_error( + response_correlation, + "platform_error", + "response_middleware_body", + ); + } return Err(error); } debug!(error = %error, "HTTP response processing failed before commitment"); @@ -662,6 +954,8 @@ where middleware.policy_name, &middleware.target, denial, + "response_middleware_body", + response_correlation, ) .await?; } else { @@ -670,6 +964,8 @@ where request_method, middleware.policy_name, &middleware.target, + "response_middleware_body", + response_correlation, ) .await?; } @@ -707,6 +1003,19 @@ where } crate::telemetry::emit_decision(events, None); if committed { + if error.denial.is_some() { + record_committed_response_error( + response_correlation, + "denied", + "response_middleware_finish", + ); + } else { + record_committed_response_error( + response_correlation, + "platform_error", + "response_middleware_finish", + ); + } return Err(miette!( "HTTP response middleware failed after commitment: {error}" )); @@ -719,6 +1028,8 @@ where middleware.policy_name, &middleware.target, denial, + "response_middleware_finish", + response_correlation, ) .await?; } else { @@ -727,6 +1038,8 @@ where request_method, middleware.policy_name, &middleware.target, + "response_middleware_finish", + response_correlation, ) .await?; } @@ -779,6 +1092,7 @@ where server_wants_close, &trailer_names, ); + let head = final_response_head(&head, response_correlation); client.write_all(&head).await.into_diagnostic()?; if matches!(framing, ResponseFraming::Chunked) { for unit in &finish.body_units { @@ -803,6 +1117,7 @@ where } } client.flush().await.into_diagnostic()?; + record_allowed_response(response_correlation); Ok(Some( if server_wants_close || (committed && close_delimited_output) @@ -829,6 +1144,7 @@ async fn relay_headers_only_response( body_length: BodyLength, server_wants_close: bool, event_stream: bool, + response_correlation: Option<&RequestCorrelation>, ) -> Result where U: AsyncRead + Unpin, @@ -842,6 +1158,7 @@ where server_wants_close, declared_trailers, ); + let head = final_response_head(&head, response_correlation); client.write_all(&head).await.into_diagnostic()?; if is_bodiless_response(request_method, status_code) { @@ -1285,7 +1602,8 @@ fn validate_http_field_value(value: &str) -> Result<()> { } fn is_protected_response_field(name: &str) -> bool { - name.eq_ignore_ascii_case("content-length") + name.eq_ignore_ascii_case(openshell_core::token_grant_failure::REQUEST_ID_HEADER) + || name.eq_ignore_ascii_case("content-length") || is_hidden_response_field(name) || openshell_supervisor_middleware::headers::is_response_credential_header(name) } @@ -2022,7 +2340,10 @@ async fn send_response_middleware_denial( policy_name: &str, target: &HttpRequestTarget, denial: &openshell_supervisor_middleware::MiddlewareDenial, + stage: &'static str, + correlation: Option<&RequestCorrelation>, ) -> Result<()> { + record_denied_response(correlation, stage); let mut body = serde_json::Map::new(); body.insert("error".into(), serde_json::json!("middleware_denied")); body.insert( @@ -2047,7 +2368,8 @@ async fn send_response_middleware_denial( "HTTP/1.1 403 Forbidden\r\nContent-Type: application/json\r\nContent-Length: {}\r\nX-OpenShell-Policy: {policy_name}\r\nConnection: close\r\n\r\n", body.len() ); - client.write_all(head.as_bytes()).await.into_diagnostic()?; + let head = final_response_head(head.as_bytes(), correlation); + client.write_all(&head).await.into_diagnostic()?; if !request_method.eq_ignore_ascii_case("HEAD") { client.write_all(&body).await.into_diagnostic()?; } @@ -2060,7 +2382,10 @@ async fn send_response_delivery_failure( request_method: &str, policy_name: &str, target: &HttpRequestTarget, + stage: &'static str, + correlation: Option<&RequestCorrelation>, ) -> Result<()> { + record_platform_response_error(correlation, stage); let body = serde_json::to_vec(&serde_json::json!({ "error": "response_delivery_failed", "detail": "The upstream request may have completed, but OpenShell could not deliver its response. Retrying may repeat upstream side effects.", @@ -2076,7 +2401,8 @@ async fn send_response_delivery_failure( "HTTP/1.1 502 Bad Gateway\r\nContent-Type: application/json\r\nContent-Length: {}\r\nX-OpenShell-Policy: {policy_name}\r\nConnection: close\r\n\r\n", body.len() ); - client.write_all(head.as_bytes()).await.into_diagnostic()?; + let head = final_response_head(head.as_bytes(), correlation); + client.write_all(&head).await.into_diagnostic()?; if !request_method.eq_ignore_ascii_case("HEAD") { client.write_all(&body).await.into_diagnostic()?; } @@ -2197,7 +2523,7 @@ mod tests { let (events, received) = exported("warn", || { super::emit_http_response_middleware_failure( "response-policy", - &super::HttpRequestTarget::default(), + &HttpRequestTarget::default(), 200, committed, ); @@ -2231,7 +2557,7 @@ mod tests { }; super::emit_http_response_diagnostics( "response-policy", - &super::HttpRequestTarget::default(), + &HttpRequestTarget::default(), 200, &diagnostics, ); @@ -2258,7 +2584,7 @@ mod tests { #[test] fn fail_closed_invocations_and_enforced_failures_record_the_denial() { - let target = super::HttpRequestTarget::default(); + let target = HttpRequestTarget::default(); for (outcome, failed, action_id) in [ (Outcome::BlockDelivery, false, 2), (Outcome::FailClosed, true, 2), @@ -2287,11 +2613,730 @@ mod tests { } use super::{ - ResponseFraming, parse_connection_keep_alive, parse_response_head_for_middleware, - response_is_http_10, serialize_response_head, + DEFAULT_HTTP_RESPONSE_WHOLE_BODY_TIMEOUT, HttpResponseMiddlewareRelay, + RelayResponseOptions, ResponseFraming, parse_connection_keep_alive, + parse_response_head_for_middleware, relay_response, response_is_http_10, + serialize_response_head, validate_http_response_header_block, + }; + use crate::l7::provider::{BodyLength, L7Request, RelayOutcome}; + use crate::l7::relay::L7EvalContext; + use openshell_core::proto::{ + Decision, HttpHeader, HttpRequestResult, HttpRequestTarget, HttpResponseBlockDelivery, + HttpResponseBodyMode, HttpResponseBodyResult, HttpResponseBodyTransform, HttpResponseEvent, + HttpResponseEventResult, HttpResponsePreflightInspect, HttpResponsePreflightResult, + HttpResponseTrailersResult, MiddlewareBinding, MiddlewareManifest, RequestContext, + SupervisorMiddlewareOperation, SupervisorMiddlewarePhase, http_response_body_result, + http_response_body_transform, http_response_body_unit, http_response_event, + http_response_event_result, http_response_preflight_result, }; - use crate::l7::provider::BodyLength; - use openshell_core::proto::HttpHeader; + use std::collections::HashMap; + use std::sync::Arc; + use tokio::sync::mpsc; + use tokio_stream::wrappers::ReceiverStream; + + struct StreamingResponseMiddleware; + + struct BlockingResponseMiddleware; + + #[tonic::async_trait] + impl openshell_supervisor_middleware::InProcessMiddleware for BlockingResponseMiddleware { + async fn describe(&self) -> MiddlewareManifest { + MiddlewareManifest { + name: "test/blocking-response".into(), + service_version: "test".into(), + bindings: vec![MiddlewareBinding { + operation: SupervisorMiddlewareOperation::HttpResponse as i32, + phase: SupervisorMiddlewarePhase::PreReturn as i32, + max_payload_bytes: 4096, + request_timeout: None, + }], + expected_audience: String::new(), + extension: Some(openshell_core::extension_protocol::extension_metadata( + openshell_core::extension_protocol::ExtensionFamily::SupervisorMiddleware, + "openshell/test-blocking-response", + "test", + [], + )), + } + } + + async fn validate_config( + &self, + _middleware_name: &str, + _config: &prost_types::Struct, + ) -> miette::Result<()> { + Ok(()) + } + + async fn evaluate_http_request( + &self, + _request: openshell_supervisor_middleware::HttpRequestView<'_>, + ) -> miette::Result { + Ok(HttpRequestResult { + decision: Decision::Allow as i32, + ..Default::default() + }) + } + + async fn open_http_response_pre_return( + &self, + mut requests: mpsc::Receiver, + ) -> Result + { + let first = requests + .recv() + .await + .ok_or_else(|| tonic::Status::invalid_argument("missing response preflight"))?; + if !matches!(first.event, Some(http_response_event::Event::Preflight(_))) { + return Err(tonic::Status::invalid_argument( + "expected response preflight", + )); + } + let result = HttpResponseEventResult { + result: Some(http_response_event_result::Result::PreflightResult( + HttpResponsePreflightResult { + action: Some(http_response_preflight_result::Action::BlockDelivery( + HttpResponseBlockDelivery {}, + )), + reason_code: "test_block".into(), + ..Default::default() + }, + )), + }; + Ok(Box::pin(futures::stream::iter([Ok(result)]))) + } + } + + #[tonic::async_trait] + impl openshell_supervisor_middleware::InProcessMiddleware for StreamingResponseMiddleware { + async fn describe(&self) -> MiddlewareManifest { + MiddlewareManifest { + name: "test/streaming-response".into(), + service_version: "test".into(), + bindings: vec![MiddlewareBinding { + operation: SupervisorMiddlewareOperation::HttpResponse as i32, + phase: SupervisorMiddlewarePhase::PreReturn as i32, + max_payload_bytes: 4096, + request_timeout: None, + }], + expected_audience: String::new(), + extension: Some(openshell_core::extension_protocol::extension_metadata( + openshell_core::extension_protocol::ExtensionFamily::SupervisorMiddleware, + "openshell/test-streaming-response", + "test", + [], + )), + } + } + + async fn validate_config( + &self, + _middleware_name: &str, + _config: &prost_types::Struct, + ) -> miette::Result<()> { + Ok(()) + } + + async fn evaluate_http_request( + &self, + _request: openshell_supervisor_middleware::HttpRequestView<'_>, + ) -> miette::Result { + Ok(HttpRequestResult { + decision: Decision::Allow as i32, + ..Default::default() + }) + } + + async fn open_http_response_pre_return( + &self, + mut requests: mpsc::Receiver, + ) -> Result + { + let (sender, receiver) = mpsc::channel(4); + tokio::spawn(async move { + while let Some(event) = requests.recv().await { + let Some(event) = event.event else { + break; + }; + let result = match event { + http_response_event::Event::Preflight(_) => HttpResponseEventResult { + result: Some(http_response_event_result::Result::PreflightResult( + HttpResponsePreflightResult { + action: Some(http_response_preflight_result::Action::Inspect( + HttpResponsePreflightInspect { + body_mode: HttpResponseBodyMode::StreamBytes as i32, + header_mutations: Vec::new(), + }, + )), + ..Default::default() + }, + )), + }, + http_response_event::Event::Body(body) => { + let Some(http_response_body_unit::Payload::Data(data)) = body.payload + else { + break; + }; + HttpResponseEventResult { + result: Some(http_response_event_result::Result::BodyResult( + HttpResponseBodyResult { + sequence: body.sequence, + action: Some(http_response_body_result::Action::Transform( + HttpResponseBodyTransform { + replacement: Some( + http_response_body_transform::Replacement::Data( + data, + ), + ), + }, + )), + ..Default::default() + }, + )), + } + } + http_response_event::Event::Trailers(_) => HttpResponseEventResult { + result: Some(http_response_event_result::Result::TrailersResult( + HttpResponseTrailersResult::default(), + )), + }, + http_response_event::Event::SessionEnd(_) => break, + }; + if sender.send(Ok(result)).await.is_err() { + break; + } + } + }); + Ok(Box::pin(ReceiverStream::new(receiver))) + } + } + + fn test_correlation_for_method( + request_id: &str, + method: &str, + ) -> crate::l7::correlation::RequestCorrelation { + let request = L7Request { + action: method.into(), + target: "/data".into(), + query_params: HashMap::new(), + raw_header: b"GET /data HTTP/1.1\r\nHost: example.test\r\n\r\n".to_vec(), + body_length: BodyLength::None, + }; + crate::l7::correlation::RequestCorrelation::with_request_id( + &request, + &L7EvalContext { + host: "example.test".into(), + port: 443, + ..Default::default() + }, + request_id.into(), + ) + } + + fn test_correlation(request_id: &str) -> crate::l7::correlation::RequestCorrelation { + test_correlation_for_method(request_id, "GET") + } + + fn request_id_header_count(response: &[u8]) -> usize { + String::from_utf8_lossy(response) + .lines() + .filter(|line| { + line.split_once(':').is_some_and(|(name, _)| { + name.eq_ignore_ascii_case( + openshell_core::token_grant_failure::REQUEST_ID_HEADER, + ) + }) + }) + .count() + } + + #[tokio::test] + async fn allowed_final_response_carries_platform_receipt() { + let correlation = test_correlation("receipt-1"); + let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nbody"[..]; + let mut client = Vec::new(); + + let outcome = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect("relay response"); + + assert!(matches!(outcome, RelayOutcome::Reusable)); + assert_eq!(request_id_header_count(&client), 1); + assert!(String::from_utf8_lossy(&client).contains("x-openshell-request-id: receipt-1\r\n")); + } + + #[tokio::test] + async fn response_middleware_denial_records_that_upstream_was_reached() { + use tracing_subscriber::layer::SubscriberExt as _; + + let exporter = opentelemetry_sdk::trace::InMemorySpanExporterBuilder::new().build(); + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let subscriber = tracing_subscriber::registry().with(openshell_otel::layer( + &provider, + "response-correlation-test", + )); + let correlation = tracing::subscriber::with_default(subscriber, || { + test_correlation_for_method("response-denied", "POST") + }); + let runner = + openshell_supervisor_middleware::ChainRunner::new(Arc::new(BlockingResponseMiddleware)); + let chain = vec![openshell_supervisor_middleware::ChainEntry { + name: "response".into(), + implementation: "test/blocking-response".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: openshell_supervisor_middleware::OnError::FailClosed, + }]; + let middleware = HttpResponseMiddlewareRelay { + chain: &chain, + runner: &runner, + request_context: RequestContext { + request_id: "response-denied".into(), + sandbox_id: "sandbox-1".into(), + ..Default::default() + }, + target: HttpRequestTarget { + scheme: "https".into(), + host: "example.test".into(), + port: 443, + method: "POST".into(), + path: "/mutate".into(), + query: String::new(), + }, + policy_name: "test-policy", + generation_guard: None, + whole_body_timeout: DEFAULT_HTTP_RESPONSE_WHOLE_BODY_TIMEOUT, + }; + let mut upstream = &b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"[..]; + let mut client = Vec::new(); + + let outcome = relay_response( + "POST", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + Some(middleware), + ) + .await + .expect("middleware denial response"); + + assert!(matches!(outcome, RelayOutcome::Consumed)); + assert!(client.starts_with(b"HTTP/1.1 403 Forbidden\r\n")); + assert!( + String::from_utf8_lossy(&client) + .contains("x-openshell-request-id: response-denied\r\n") + ); + drop(correlation); + + let span = exporter + .get_finished_spans() + .expect("finished spans") + .into_iter() + .find(|span| span.name == "openshell.http.request") + .expect("HTTP enforcement span"); + assert!(matches!( + span.status, + opentelemetry::trace::Status::Error { .. } + )); + let attributes = span + .attributes + .into_iter() + .map(|attribute| { + ( + attribute.key.as_str().to_string(), + attribute.value.to_string(), + ) + }) + .collect::>(); + assert_eq!(attributes["openshell.target.reached"], "true"); + assert_eq!(attributes["openshell.http.outcome"], "denied"); + assert_eq!(attributes["http.response.status_code"], "403"); + assert!(!attributes.contains_key("openshell.error.stage")); + } + + #[tokio::test] + async fn upstream_receipt_is_replaced_by_platform_receipt() { + let correlation = test_correlation("trusted"); + let mut upstream = + &b"HTTP/1.1 200 OK\r\nX-OpenShell-Request-ID: forged\r\nContent-Length: 0\r\n\r\n"[..]; + let mut client = Vec::new(); + + relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect("relay response"); + + assert!(!String::from_utf8_lossy(&client).contains("forged")); + assert!(String::from_utf8_lossy(&client).contains("x-openshell-request-id: trusted\r\n")); + assert_eq!(request_id_header_count(&client), 1); + } + + #[tokio::test] + async fn ordinary_response_removes_receipt_connection_nomination() { + let correlation = test_correlation("trusted"); + let mut upstream = &b"HTTP/1.1 200 OK\r\n\ + Connection: X-OpenShell-Request-ID\r\n\ + cOnNeCtIoN: keep-alive, x-OpEnShElL-rEqUeSt-Id\r\n\ + X-OpenShell-Request-ID: forged-head\r\n\ + Trailer: x-openshell-request-id, X-Checksum\r\n\ + Transfer-Encoding: chunked\r\n\r\n\ + 4\r\nbody\r\n\ + 0\r\n\ + x-OPENshell-request-ID: forged-trailer\r\n\ + X-Checksum: ok\r\n\r\n"[..]; + let mut client = Vec::new(); + + let outcome = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect("relay response"); + + assert!(matches!(outcome, RelayOutcome::Reusable)); + let delivered = String::from_utf8(client).expect("HTTP response is UTF-8"); + let head = delivered + .split_once("\r\n\r\n") + .expect("response head terminator") + .0; + let connection_headers: Vec<_> = head + .lines() + .filter(|line| { + line.split_once(':') + .is_some_and(|(name, _)| name.eq_ignore_ascii_case("connection")) + }) + .collect(); + assert_eq!(connection_headers, ["cOnNeCtIoN: keep-alive"]); + assert!(!delivered.contains("forged"), "{delivered}"); + assert!(head.contains("Trailer: X-Checksum"), "{head}"); + assert!(delivered.contains("0\r\nX-Checksum: ok\r\n\r\n")); + assert!(head.contains("x-openshell-request-id: trusted"), "{head}"); + assert_eq!(request_id_header_count(delivered.as_bytes()), 1); + } + + #[tokio::test] + async fn ordinary_response_rejects_folded_receipt_connection_nomination() { + let correlation = test_correlation("trusted"); + let mut upstream = &b"HTTP/1.1 200 OK\r\nConnection: keep-alive,\r\n X-OpenShell-Request-ID\r\nContent-Length: 0\r\n\r\n"[..]; + let mut client = Vec::new(); + + let error = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect_err("folded response fields must fail closed"); + + assert!( + error + .to_string() + .contains("response header continuation lines are not supported"), + "{error}" + ); + assert!(client.is_empty()); + } + + #[tokio::test] + async fn ordinary_response_rejects_bare_lf_receipt_smuggling() { + let correlation = test_correlation("trusted"); + let mut upstream = &b"HTTP/1.1 200 OK\r\nX-Safe: yes\nX-OpenShell-Request-ID: forged\r\nContent-Length: 0\r\n\r\n"[..]; + let mut client = Vec::new(); + + let error = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect_err("bare LF response fields must fail closed"); + + assert!(error.to_string().contains("bare LF"), "{error}"); + assert!(client.is_empty()); + } + + #[test] + fn response_head_validation_rejects_malformed_syntax() { + let cases: &[(&str, &[u8])] = &[ + ("incomplete block", b"HTTP/1.1 200 OK\r\nX-Test: yes\r\n"), + ("bare LF", b"HTTP/1.1 200 OK\r\nX-Test: yes\n\r\n"), + ("bare CR", b"HTTP/1.1 200 OK\r\nX-Test: yes\rX: no\r\n\r\n"), + ( + "obsolete fold", + b"HTTP/1.1 200 OK\r\nX-Test: yes\r\n continued\r\n\r\n", + ), + ("unsupported protocol", b"HTTP/2 200 OK\r\n\r\n"), + ("invalid status range", b"HTTP/1.1 099 Nope\r\n\r\n"), + ("non-numeric status", b"HTTP/1.1 2O0 Nope\r\n\r\n"), + ("four-digit status", b"HTTP/1.1 2000 Nope\r\n\r\n"), + ("missing reason separator", b"HTTP/1.1 200\r\n\r\n"), + ("malformed field", b"HTTP/1.1 200 OK\r\nX-Test\r\n\r\n"), + ("empty field name", b"HTTP/1.1 200 OK\r\n: value\r\n\r\n"), + ( + "non-token field name", + b"HTTP/1.1 200 OK\r\nBad Name: value\r\n\r\n", + ), + ( + "whitespace before colon", + b"HTTP/1.1 200 OK\r\nX-Test : value\r\n\r\n", + ), + ( + "field control byte", + b"HTTP/1.1 200 OK\r\nX-Test: value\x0bmore\r\n\r\n", + ), + ( + "field delete byte", + b"HTTP/1.1 200 OK\r\nX-Test: value\x7f\r\n\r\n", + ), + ("reason control byte", b"HTTP/1.1 200 O\x00K\r\n\r\n"), + ( + "early empty line", + b"HTTP/1.1 200 OK\r\n\r\nX-Smuggled: yes\r\n\r\n", + ), + ]; + + for (name, input) in cases { + assert!( + validate_http_response_header_block(input).is_err(), + "{name} unexpectedly passed validation" + ); + } + } + + #[test] + fn response_head_validation_accepts_supported_informational_and_upgrade_responses() { + for response in [ + &b"HTTP/1.0 200 OK\r\nX-Test: yes\r\n\r\n"[..], + &b"HTTP/1.1 103 Early Hints\r\nLink: ; rel=preload\r\n\r\n"[..], + &b"HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n"[..], + ] { + validate_http_response_header_block(response).expect("valid HTTP/1.x response head"); + } + } + + #[tokio::test] + async fn response_middleware_rejects_forged_request_id_trailer() { + let correlation = test_correlation("trusted"); + let runner = openshell_supervisor_middleware::ChainRunner::new(Arc::new( + StreamingResponseMiddleware, + )); + let chain = vec![openshell_supervisor_middleware::ChainEntry { + name: "response".into(), + implementation: "test/streaming-response".into(), + order: 0, + config: prost_types::Struct::default(), + on_error: openshell_supervisor_middleware::OnError::FailClosed, + }]; + let middleware = HttpResponseMiddlewareRelay { + chain: &chain, + runner: &runner, + request_context: RequestContext { + request_id: "middleware-request".into(), + sandbox_id: "sandbox-1".into(), + ..Default::default() + }, + target: HttpRequestTarget { + scheme: "https".into(), + host: "example.test".into(), + port: 443, + method: "GET".into(), + path: "/data".into(), + query: String::new(), + }, + policy_name: "test-policy", + generation_guard: None, + whole_body_timeout: DEFAULT_HTTP_RESPONSE_WHOLE_BODY_TIMEOUT, + }; + let mut upstream = &b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nbody\r\n0\r\nX-OpenShell-Request-ID: forged\r\n\r\n"[..]; + let mut client = Vec::new(); + + let error = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + Some(middleware), + ) + .await + .expect_err("platform correlation trailers must fail closed"); + + assert!( + error.to_string().contains("protected field name"), + "{error}" + ); + let delivered = String::from_utf8_lossy(&client); + assert!(!delivered.contains("forged"), "{delivered}"); + assert!( + delivered.contains("x-openshell-request-id: trusted\r\n"), + "{delivered}" + ); + assert_eq!(request_id_header_count(&client), 1); + } + + #[test] + fn response_middleware_rejects_declared_request_id_trailer() { + let error = parse_response_head_for_middleware( + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nTrailer: X-OpenShell-Request-ID\r\n\r\n", + ) + .expect_err("platform correlation trailer declarations must fail closed"); + + assert!( + error.to_string().contains("protected trailer field"), + "{error}" + ); + } + + #[tokio::test] + async fn informational_responses_are_not_decorated_but_upgrade_receives_a_receipt() { + let correlation = test_correlation("receipt-1"); + let mut upstream = + &b"HTTP/1.1 103 Early Hints\r\n\r\nHTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"[..]; + let mut client = Vec::new(); + + relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect("relay response"); + assert!(client.starts_with(b"HTTP/1.1 103 Early Hints\r\n\r\nHTTP/1.1 200 OK\r\n")); + assert_eq!(request_id_header_count(&client), 1); + + let mut upstream = &b"HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nX-OpenShell-Request-ID: forged\r\n\r\n"[..]; + let mut client = Vec::new(); + let outcome = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect("relay response"); + assert!(matches!(outcome, RelayOutcome::Upgraded { .. })); + assert_eq!(request_id_header_count(&client), 1); + assert!(!String::from_utf8_lossy(&client).contains("forged")); + assert!(String::from_utf8_lossy(&client).contains("x-openshell-request-id: receipt-1\r\n")); + } + + #[tokio::test] + async fn switching_protocols_removes_receipt_connection_nomination() { + let correlation = test_correlation("trusted-upgrade"); + let mut upstream = &b"HTTP/1.1 101 Switching Protocols\r\n\ + Connection: X-OpenShell-Request-ID\r\n\ + cOnNeCtIoN: keep-alive, x-OpEnShElL-rEqUeSt-Id, UpGrAdE\r\n\ + Upgrade: websocket\r\n\ + X-OpenShell-Request-ID: forged\r\n\r\n"[..]; + let mut client = Vec::new(); + + let outcome = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect("relay response"); + + assert!(matches!(outcome, RelayOutcome::Upgraded { .. })); + let delivered = String::from_utf8(client).expect("HTTP response is UTF-8"); + let connection_headers: Vec<_> = delivered + .lines() + .filter(|line| { + line.split_once(':') + .is_some_and(|(name, _)| name.eq_ignore_ascii_case("connection")) + }) + .collect(); + assert_eq!(connection_headers, ["cOnNeCtIoN: keep-alive, UpGrAdE"]); + assert!(!delivered.contains("forged"), "{delivered}"); + assert!( + delivered.contains("x-openshell-request-id: trusted-upgrade\r\n"), + "{delivered}" + ); + assert_eq!(request_id_header_count(delivered.as_bytes()), 1); + } + + #[tokio::test] + async fn switching_protocols_rejects_folded_receipt_connection_nomination() { + let correlation = test_correlation("trusted-upgrade"); + let mut upstream = &b"HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade,\r\n\tX-OpenShell-Request-ID\r\nUpgrade: websocket\r\n\r\n"[..]; + let mut client = Vec::new(); + + let error = relay_response( + "GET", + &mut upstream, + &mut client, + RelayResponseOptions { + correlation: Some(&correlation), + ..Default::default() + }, + None, + ) + .await + .expect_err("folded upgrade fields must fail closed"); + + assert!( + error + .to_string() + .contains("response header continuation lines are not supported"), + "{error}" + ); + assert!(client.is_empty()); + } #[test] fn http_10_connection_persistence() { diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 96c0b8f50a..b79c0caa60 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -253,6 +253,7 @@ fn emit_credential_endpoint_mismatch( port: u16, policy_name: &str, connect_span: &tracing::Span, + correlation: Option<&crate::l7::correlation::RequestCorrelation>, ) { let event = build_credential_endpoint_mismatch_event(method, host, port, policy_name); let finding = crate::l7::build_credential_endpoint_mismatch_finding( @@ -261,7 +262,12 @@ fn emit_credential_endpoint_mismatch( None, "Provider credential endpoint binding mismatch; request denied", ); - crate::telemetry::emit_decision([event, finding], Some(connect_span)); + if let Some(correlation) = correlation { + correlation.emit(event); + correlation.emit(finding); + } else { + crate::telemetry::emit_decision([event, finding], Some(connect_span)); + } } /// Hostnames injected by compute drivers as `/etc/hosts` aliases for the host @@ -2120,13 +2126,18 @@ fn emit_forward_l7_decision( decision: openshell_ocsf::OcsfEvent, evaluation_error: Option, connect_span: &tracing::Span, + correlation: Option<&crate::l7::correlation::RequestCorrelation>, ) { // The HTTP decision carries policy context that its evaluation-error // companion lacks, so it must supply the shared deny span's attributes. - crate::telemetry::emit_decision( - std::iter::once(decision).chain(evaluation_error), - Some(connect_span), - ); + let events = std::iter::once(decision).chain(evaluation_error); + if let Some(correlation) = correlation { + for event in events { + correlation.emit(event); + } + return; + } + crate::telemetry::emit_decision(events, Some(connect_span)); } fn build_forward_parse_error_ocsf_event( @@ -2437,17 +2448,25 @@ async fn deny_forward_destination( decision: &EgressDecision, denial_tx: Option<&mpsc::UnboundedSender>, activity_tx: Option<&ActivitySender>, + correlation: Option<&crate::l7::correlation::RequestCorrelation>, ) -> Result<()> where C: TokioAsyncWrite + Unpin, { let detail = destination_denial_detail(denial.kind); - crate::telemetry::emit_decision( - [build_forward_destination_deny_ocsf_event( - denial, peer_addr, method, host, port, path, binary, pid, ancestors, cmdline, policy, - )], - Some(connect_span), + let event = build_forward_destination_deny_ocsf_event( + denial, peer_addr, method, host, port, path, binary, pid, ancestors, cmdline, policy, ); + if let Some(correlation) = correlation { + correlation.record_policy_decision("deny"); + correlation.record_enforcement_action("block"); + correlation.record_final_http_outcome("denied"); + correlation.record_response_status(403); + correlation.record_error_stage("destination_policy"); + correlation.emit(event); + } else { + crate::telemetry::emit_decision([event], Some(connect_span)); + } emit_denial_simple( denial_tx, @@ -2465,11 +2484,12 @@ where } respond( client, - &build_json_error_response( + &build_json_error_response_with_request_id( 403, "Forbidden", "ssrf_denied", &format!("{method} {host}:{port} blocked: {detail}"), + correlation.map(crate::l7::correlation::RequestCorrelation::request_id), ), ) .await @@ -5402,6 +5422,7 @@ struct ForwardResponseMiddleware<'a> { exchange: &'a crate::l7::middleware::HttpMiddlewareExchange, } +#[cfg(test)] async fn relay_rewritten_forward_request( method: &str, path: &str, @@ -5410,6 +5431,25 @@ async fn relay_rewritten_forward_request( upstream: &mut U, options: ForwardRelayOptions<'_>, ) -> Result +where + C: TokioAsyncRead + TokioAsyncWrite + Unpin, + U: TokioAsyncRead + TokioAsyncWrite + Unpin, +{ + relay_rewritten_forward_request_correlated( + method, path, rewritten, client, upstream, options, None, + ) + .await +} + +async fn relay_rewritten_forward_request_correlated( + method: &str, + path: &str, + rewritten: Vec, + client: &mut C, + upstream: &mut U, + options: ForwardRelayOptions<'_>, + correlation: Option<&crate::l7::correlation::RequestCorrelation>, +) -> Result where C: TokioAsyncRead + TokioAsyncWrite + Unpin, U: TokioAsyncRead + TokioAsyncWrite + Unpin, @@ -5435,7 +5475,7 @@ where .response_relay(&req, middleware.ctx, middleware.scheme) }); - crate::l7::rest::relay_http_request_with_response_middleware_guarded_observed( + crate::l7::rest::relay_http_request_with_response_middleware_guarded_observed_correlated( &req, client, upstream, @@ -5456,6 +5496,7 @@ where }, response_middleware, options.endpoint_observer, + correlation, ) .await } @@ -5871,6 +5912,19 @@ async fn handle_forward_proxy( "https" => Some(443), _ => None, }; + let correlation_request = crate::l7::rest::request_from_buffered_http( + method, + path.clone(), + &upstream_target, + forward_request_bytes, + )?; + let correlation = + crate::l7::correlation::RequestCorrelation::new(&correlation_request, &l7_ctx); + let mut audited_client = correlation.observe_response(client); + let client = &mut audited_client; + correlation.record_policy_decision("allow"); + correlation.record_enforcement_action("allow"); + forward_request_bytes = correlation_request.raw_header; if let Some(route) = decision .endpoint .l7_route @@ -5897,13 +5951,18 @@ async fn handle_forward_proxy( ), ); emit_activity_simple(activity_tx, true, "policy_stale"); + correlation.record_enforcement_action("block"); + correlation.record_final_http_outcome("denied"); + correlation.record_response_status(403); + correlation.record_error_stage("policy_generation"); respond( client, - &build_json_error_response( + &build_correlated_json_error_response( 403, "Forbidden", "policy_denied", &format!("{method} {host_lc}:{port}{telemetry_path} not permitted by policy"), + correlation.request_id(), ), ) .await?; @@ -5922,15 +5981,20 @@ async fn handle_forward_proxy( ); emit_l7_tunnel_close_after_policy_change(&host_lc, port, e); emit_activity_simple(activity_tx, true, "policy_stale"); + correlation.record_enforcement_action("block"); + correlation.record_final_http_outcome("denied"); + correlation.record_response_status(403); + correlation.record_error_stage("policy_generation"); respond( client, - &build_json_error_response( + &build_correlated_json_error_response( 403, "Forbidden", "policy_denied", &format!( "{method} {host_lc}:{port}{telemetry_path} not permitted by policy" ), + correlation.request_id(), ), ) .await?; @@ -5939,29 +6003,64 @@ async fn handle_forward_proxy( }; let Ok(redacted_path) = secrets::redact_target_for_policy(&path) else { + correlation.record_final_http_outcome("platform_error"); + correlation.record_response_status(400); + correlation.record_error_stage("credential_placeholder"); respond( client, - &build_json_error_response( + &build_correlated_json_error_response( 400, "Bad Request", "invalid_credential_placeholder", "request-target contains an invalid credential placeholder", + correlation.request_id(), ), ) .await?; return Ok(()); }; let Some(l7_config) = select_l7_config_for_path(&route.configs, &redacted_path) else { + let reason = "no L7 endpoint path matched request"; + correlation.record_policy_decision("deny"); + correlation.record_enforcement_action("block"); + correlation.record_final_http_outcome("denied"); + correlation.record_response_status(403); + correlation.emit( + HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::for_http_method(method)) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .http_request(HttpRequest::new( + method, + OcsfUrl::new("http", &host_lc, &telemetry_path, port), + )) + .http_response(HttpResponse { code: 403 }) + .dst_endpoint(Endpoint::from_domain(&host_lc, port)) + .src_endpoint(Endpoint::from_ip(workload_addr.ip(), workload_addr.port())) + .actor_process( + Process::from_bypass(&binary_str, &pid_str, &ancestors_str) + .with_cmd_line(&cmdline_str), + ) + .firewall_rule(policy_str, "l7") + .message(format!( + "FORWARD_L7 denied {method} {host_lc}:{port}{telemetry_path}" + )) + .status_detail(reason) + .build(), + ); emit_activity_simple(activity_tx, true, "l7_policy"); respond( client, - &build_json_error_response( + &build_correlated_json_error_response( 403, "Forbidden", "policy_denied", &format!( "{method} {host_lc}:{port}{telemetry_path} did not match an L7 endpoint path" ), + correlation.request_id(), ), ) .await?; @@ -5984,30 +6083,33 @@ async fn handle_forward_proxy( if !l7_config.config.allow_encoded_slash && crate::l7::path::canonical_path_has_encoded_slash(&path) { - crate::telemetry::emit_decision( - [build_forward_l7_parse_rejection_ocsf_event( - workload_addr, - method, - &host_lc, - port, - &telemetry_path, - &binary_str, - &pid_str, - &ancestors_str, - &cmdline_str, - policy_str, - FORWARD_ENCODED_SLASH_REJECTION_DETAIL, - )], - Some(&connect_span), - ); + correlation.record_policy_decision("deny"); + correlation.record_enforcement_action("block"); + correlation.record_final_http_outcome("denied"); + correlation.record_response_status(400); + correlation.record_error_stage("request_target"); + correlation.emit(build_forward_l7_parse_rejection_ocsf_event( + workload_addr, + method, + &host_lc, + port, + &telemetry_path, + &binary_str, + &pid_str, + &ancestors_str, + &cmdline_str, + policy_str, + FORWARD_ENCODED_SLASH_REJECTION_DETAIL, + )); emit_activity_simple(activity_tx, true, "forward_parse_rejection"); respond( client, - &build_json_error_response( + &build_correlated_json_error_response( 400, "Bad Request", "invalid_request_target", "request-target must be canonical", + correlation.request_id(), ), ) .await?; @@ -6019,6 +6121,11 @@ async fn handle_forward_proxy( &forward_request_bytes, l7_config.config.protocol, ) { + correlation.record_policy_decision("deny"); + correlation.record_enforcement_action("block"); + correlation.record_final_http_outcome("denied"); + correlation.record_response_status(403); + correlation.record_error_stage("protocol_upgrade"); if let Some(observer) = endpoint_observer.as_ref() { observer.observe(EndpointResult::PolicyDenied); } @@ -6044,7 +6151,7 @@ async fn handle_forward_proxy( )) .status_detail(upgrade_detail) .build(); - crate::telemetry::emit_decision([event], Some(&connect_span)); + correlation.emit(event); emit_activity_simple(activity_tx, true, "l7_parse_rejection"); emit_denial_simple( denial_tx, @@ -6057,11 +6164,12 @@ async fn handle_forward_proxy( ); respond( client, - &build_json_error_response( + &build_correlated_json_error_response( 403, "Forbidden", "unsupported_l7_protocol", upgrade_detail, + correlation.request_id(), ), ) .await?; @@ -6102,20 +6210,26 @@ async fn handle_forward_proxy( { Ok(info) => info, Err(e) => { + correlation.record_policy_decision("deny"); + correlation.record_enforcement_action("block"); + correlation.record_final_http_outcome("denied"); + correlation.record_response_status(400); + correlation.record_error_stage("graphql_inspection"); let event = build_forward_rejection_event( &host_lc, port, format!("FORWARD_GRAPHQL_L7 request rejected: {e}"), ); - egress_emit!(event); + correlation.emit(event); emit_activity_simple(activity_tx, true, "l7_parse_rejection"); respond( client, - &build_json_error_response( + &build_correlated_json_error_response( 400, "Bad Request", "invalid_graphql_request", &format!("GraphQL request rejected before policy evaluation: {e}"), + correlation.request_id(), ), ) .await?; @@ -6154,20 +6268,26 @@ async fn handle_forward_proxy( { Ok(body) => body, Err(e) => { + correlation.record_policy_decision("deny"); + correlation.record_enforcement_action("block"); + correlation.record_final_http_outcome("denied"); + correlation.record_response_status(400); + correlation.record_error_stage("jsonrpc_inspection"); let event = build_forward_rejection_event( &host_lc, port, format!("FORWARD_JSONRPC_L7 request rejected: {e}"), ); - egress_emit!(event); + correlation.emit(event); emit_activity_simple(activity_tx, true, "l7_parse_rejection"); respond( client, - &build_json_error_response( + &build_correlated_json_error_response( 400, "Bad Request", "invalid_jsonrpc_request", &format!("JSON-RPC request rejected before policy evaluation: {e}"), + correlation.request_id(), ), ) .await?; @@ -6181,7 +6301,7 @@ async fn handle_forward_proxy( }; // Policy evaluation must use the selected revision's inspection, // including method classification, just as the CONNECT relays do. - let Some(info) = crate::l7::relay::enforce_mcp_protocol_version( + let Some(info) = crate::l7::relay::enforce_mcp_protocol_version_correlated( &l7_config.config, &jsonrpc_request, info, @@ -6189,6 +6309,7 @@ async fn handle_forward_proxy( &l7_ctx, &telemetry_path, endpoint_observer.as_ref(), + Some(&correlation), ) .await? else { @@ -6213,7 +6334,14 @@ async fn handle_forward_proxy( let mut evaluation_error = None; let (allowed, reason) = hard_deny_reason.map_or_else( || { - crate::l7::relay::evaluate_l7_request(&tunnel_engine, &l7_ctx, &request_info) + correlation + .evaluate_policy(|| { + crate::l7::relay::evaluate_l7_request( + &tunnel_engine, + &l7_ctx, + &request_info, + ) + }) .unwrap_or_else(|e| { evaluation_error = Some(build_forward_l7_evaluation_error( &host_lc, @@ -6233,6 +6361,14 @@ async fn handle_forward_proxy( (false, crate::l7::EnforcementMode::Audit) => "audit", (false, crate::l7::EnforcementMode::Enforce) => "deny", }; + correlation.record_policy_decision(if allowed { "allow" } else { "deny" }); + correlation.record_enforcement_action(if allowed { + "allow" + } else if l7_config.config.enforcement == crate::l7::EnforcementMode::Audit && !force_deny { + "audit" + } else { + "block" + }); { let (action_id, disposition_id, severity) = match decision_str { @@ -6296,13 +6432,15 @@ async fn handle_forward_proxy( .firewall_rule(policy_str, engine_type) .message(log_message) .build(); - emit_forward_l7_decision(event, evaluation_error, &connect_span); + emit_forward_l7_decision(event, evaluation_error, &connect_span, Some(&correlation)); } let effectively_denied = force_deny || (!allowed && l7_config.config.enforcement == crate::l7::EnforcementMode::Enforce); if effectively_denied { + correlation.record_final_http_outcome("denied"); + correlation.record_response_status(403); emit_activity_simple(activity_tx, true, "l7_policy"); emit_denial_simple( denial_tx, @@ -6315,13 +6453,14 @@ async fn handle_forward_proxy( ); respond( client, - &build_json_error_response( + &build_correlated_json_error_response( 403, "Forbidden", "policy_denied", &format!( "{method} {host_lc}:{port}{telemetry_path} denied by L7 policy: {reason}" ), + correlation.request_id(), ), ) .await?; @@ -6362,6 +6501,7 @@ async fn handle_forward_proxy( &decision, denial_tx, activity_tx, + Some(&correlation), ) .await?; return Ok(()); @@ -6401,6 +6541,7 @@ async fn handle_forward_proxy( &decision, denial_tx, activity_tx, + Some(&correlation), ) .await?; return Ok(()); @@ -6418,13 +6559,18 @@ async fn handle_forward_proxy( ); emit_l7_tunnel_close_after_policy_change(&host_lc, port, e); emit_activity_simple(activity_tx, true, "policy_stale"); + correlation.record_enforcement_action("block"); + correlation.record_final_http_outcome("denied"); + correlation.record_response_status(403); + correlation.record_error_stage("policy_generation"); respond( client, - &build_json_error_response( + &build_correlated_json_error_response( 403, "Forbidden", "policy_denied", &format!("{method} {host_lc}:{port}{telemetry_path} not permitted by policy"), + correlation.request_id(), ), ) .await?; @@ -6452,19 +6598,24 @@ async fn handle_forward_proxy( generation, ), ); + correlation.record_enforcement_action("block"); + correlation.record_final_http_outcome("denied"); + correlation.record_response_status(403); + correlation.record_error_stage("policy_generation"); respond( client, - &build_json_error_response( + &build_correlated_json_error_response( 403, "Forbidden", "policy_denied", &format!("{method} {host_lc}:{port}{telemetry_path} not permitted by policy"), + correlation.request_id(), ), ) .await?; return Ok(()); } - let request_id = uuid::Uuid::new_v4().to_string(); + let request_id = correlation.request_id().to_string(); let websocket_chain = forward_websocket_request.then(|| chain.clone()); let response_selection = if chain.is_empty() { None @@ -6505,16 +6656,42 @@ async fn handle_forward_proxy( crate::l7::middleware::MiddlewareApplyResult::Allowed(request) => request.raw_header, crate::l7::middleware::MiddlewareApplyResult::Denied { denial, .. } => { emit_activity_simple(activity_tx, true, "middleware"); + correlation.record_enforcement_action("block"); + correlation.record_response_status(403); + correlation.record_final_http_outcome(if denial.is_some() { + "denied" + } else { + "platform_error" + }); + correlation.record_error_stage("request_middleware"); let response = denial.as_ref().map_or_else( - || build_middleware_failure_response(&l7_ctx.policy_name), - |denial| build_middleware_deny_response(&l7_ctx.policy_name, denial), + || { + build_middleware_failure_response( + &l7_ctx.policy_name, + Some(correlation.request_id()), + ) + }, + |denial| { + build_middleware_deny_response( + &l7_ctx.policy_name, + denial, + Some(correlation.request_id()), + ) + }, ); respond(client, &response).await?; return Ok(()); } crate::l7::middleware::MiddlewareApplyResult::AdmissionExhausted => { emit_activity_simple(activity_tx, true, "middleware"); - let response = build_middleware_unavailable_response(&l7_ctx.policy_name); + correlation.record_enforcement_action("block"); + correlation.record_response_status(503); + correlation.record_final_http_outcome("platform_error"); + correlation.record_error_stage("middleware_admission"); + let response = build_middleware_unavailable_response( + &l7_ctx.policy_name, + Some(correlation.request_id()), + ); respond(client, &response).await?; return Ok(()); } @@ -6541,13 +6718,18 @@ async fn handle_forward_proxy( Ok(preflight) => preflight, Err(error) => { warn!(error = %error, "Plaintext WebSocket middleware preflight failed"); + correlation.record_enforcement_action("block"); + correlation.record_response_status(502); + correlation.record_final_http_outcome("platform_error"); + correlation.record_error_stage("websocket_middleware"); respond( client, - &build_json_error_response( + &build_correlated_json_error_response( 502, "Bad Gateway", "middleware_failed", "WebSocket middleware preflight failed", + correlation.request_id(), ), ) .await?; @@ -6556,9 +6738,28 @@ async fn handle_forward_proxy( }; crate::l7::middleware::emit_websocket_preflight_events(&l7_ctx, &preflight); if preflight.terminal_reason.is_some() { + correlation.record_enforcement_action("block"); + correlation.record_response_status(403); + correlation.record_final_http_outcome(if preflight.denial.is_some() { + "denied" + } else { + "platform_error" + }); + correlation.record_error_stage("websocket_middleware"); let response = preflight.denial.as_ref().map_or_else( - || build_middleware_failure_response(&l7_ctx.policy_name), - |denial| build_middleware_deny_response(&l7_ctx.policy_name, denial), + || { + build_middleware_failure_response( + &l7_ctx.policy_name, + Some(correlation.request_id()), + ) + }, + |denial| { + build_middleware_deny_response( + &l7_ctx.policy_name, + denial, + Some(correlation.request_id()), + ) + }, ); respond(client, &response).await?; return Ok(()); @@ -6592,6 +6793,10 @@ async fn handle_forward_proxy( Err(e) => { let refusal = e.downcast_ref::(); + correlation.record_enforcement_action("block"); + correlation.record_response_status(502); + correlation.record_final_http_outcome("platform_error"); + correlation.record_error_stage("credential_preparation"); if refusal.is_none() { warn!( dst_host = %host_lc, @@ -6605,23 +6810,13 @@ async fn handle_forward_proxy( .end(openshell_core::proto::MiddlewareSessionEndReason::Cancellation) .await; } - let response = refusal.map_or_else( - || { - build_json_error_response( - 502, - "Bad Gateway", - "token_grant_failed", - "dynamic token grant failed", - ) - }, - |refusal| { - // The refusal event already names the credential and request ID. - if let Some(observer) = endpoint_observer.as_ref() { - observer.observe_token_grant_refusal(refusal, &request_id); - } - build_token_grant_failure_response(&request_id) - }, - ); + if let Some(refusal) = refusal { + // The refusal event already names the credential and request ID. + if let Some(observer) = endpoint_observer.as_ref() { + observer.observe_token_grant_refusal(refusal, &request_id); + } + } + let response = build_token_grant_failure_response(&request_id); respond(client, &response).await?; return Ok(()); } @@ -6656,8 +6851,39 @@ async fn handle_forward_proxy( ), _ => None, }; - if let Some(guard) = credential_generation { - guard.ensure_current()?; + if let Some(guard) = credential_generation + && let Err(error) = guard.ensure_current() + { + warn!( + dst_host = %host_lc, + dst_port = port, + error = %error, + "provider credential generation changed before forward proxy write" + ); + if let Some(observer) = endpoint_observer.as_ref() { + observer.observe_credential_failure(false); + } + if let Some(session) = middleware_session.take() { + session + .end(openshell_core::proto::MiddlewareSessionEndReason::Cancellation) + .await; + } + correlation.record_enforcement_action("block"); + correlation.record_response_status(500); + correlation.record_final_http_outcome("platform_error"); + correlation.record_error_stage("credential_generation"); + respond( + client, + &build_correlated_json_error_response( + 500, + "Internal Server Error", + "credential_unavailable", + "credential material is unavailable for this request", + correlation.request_id(), + ), + ) + .await?; + return Ok(()); } // 9. Rewrite request and forward to upstream @@ -6682,11 +6908,22 @@ async fn handle_forward_proxy( .await; } if e.is_placement_mismatch() { - crate::l7::relay::reject_credential_resolution(client, &l7_ctx, method, &e) - .instrument(connect_span.clone()) - .await?; + crate::l7::relay::reject_credential_resolution( + client, + &l7_ctx, + method, + &e, + Some(&correlation), + ) + .instrument(connect_span.clone()) + .await?; client.shutdown().await.into_diagnostic()?; } else if e.is_endpoint_mismatch() { + correlation.record_policy_decision("deny"); + correlation.record_enforcement_action("block"); + correlation.record_response_status(403); + correlation.record_final_http_outcome("denied"); + correlation.record_error_stage("credential_injection"); if let Some(observer) = endpoint_observer.as_ref() { observer.observe_credential_failure(true); } @@ -6696,14 +6933,16 @@ async fn handle_forward_proxy( port, policy_str, &connect_span, + Some(&correlation), ); respond( client, - &build_json_error_response( + &build_correlated_json_error_response( 403, "Forbidden", "credential_endpoint_mismatch", "credential is not authorized for this request endpoint", + correlation.request_id(), ), ) .await?; @@ -6719,13 +6958,18 @@ async fn handle_forward_proxy( }) .await; } else { + correlation.record_enforcement_action("block"); + correlation.record_response_status(500); + correlation.record_final_http_outcome("platform_error"); + correlation.record_error_stage("credential_injection"); respond( client, - &build_json_error_response( + &build_correlated_json_error_response( 500, "Internal Server Error", "credential_injection_failed", "unresolved credential placeholder in request", + correlation.request_id(), ), ) .await?; @@ -6745,13 +6989,14 @@ async fn handle_forward_proxy( &upstream_target, rewritten, )?; - if !crate::l7::relay::enforce_final_mcp_protocol_version( + if !crate::l7::relay::enforce_final_mcp_protocol_version_correlated( config, &request, client, &l7_ctx, &telemetry_path, endpoint_observer.as_ref(), + Some(&correlation), ) .await? { @@ -6782,13 +7027,18 @@ async fn handle_forward_proxy( .end(openshell_core::proto::MiddlewareSessionEndReason::PolicyReload) .await; } + correlation.record_enforcement_action("block"); + correlation.record_final_http_outcome("denied"); + correlation.record_response_status(403); + correlation.record_error_stage("policy_generation"); respond( client, - &build_json_error_response( + &build_correlated_json_error_response( 403, "Forbidden", "policy_denied", &format!("{method} {host_lc}:{port}{telemetry_path} not permitted by policy"), + correlation.request_id(), ), ) .await?; @@ -6823,19 +7073,23 @@ async fn handle_forward_proxy( "FORWARD upstream connect failed for {host_lc}:{port}: {e}" )) .build(); - egress_emit!(event); + correlation.emit(event); if let Some(session) = middleware_session.take() { session .end(openshell_core::proto::MiddlewareSessionEndReason::UpstreamFailure) .await; } + correlation.record_response_status(502); + correlation.record_final_http_outcome("upstream_error"); + correlation.record_error_stage("upstream_connect"); respond( client, - &build_json_error_response( + &build_correlated_json_error_response( 502, "Bad Gateway", "upstream_unreachable", &format!("connection to {host_lc}:{port} failed"), + correlation.request_id(), ), ) .await?; @@ -6860,13 +7114,18 @@ async fn handle_forward_proxy( .end(openshell_core::proto::MiddlewareSessionEndReason::PolicyReload) .await; } + correlation.record_enforcement_action("block"); + correlation.record_final_http_outcome("denied"); + correlation.record_response_status(403); + correlation.record_error_stage("policy_generation"); respond( client, - &build_json_error_response( + &build_correlated_json_error_response( 403, "Forbidden", "policy_denied", &format!("{method} {host_lc}:{port}{path} not permitted by policy"), + correlation.request_id(), ), ) .await?; @@ -6884,7 +7143,7 @@ async fn handle_forward_proxy( let signing_region = forward_upgrade_config .as_ref() .map_or("", |config| config.signing_region.as_str()); - let outcome_result = relay_rewritten_forward_request( + let outcome_result = relay_rewritten_forward_request_correlated( method, &upstream_target, rewritten, @@ -6912,6 +7171,7 @@ async fn handle_forward_proxy( }), endpoint_observer: endpoint_observer.as_ref(), }, + Some(&correlation), ) .await; let outcome_result = match outcome_result { @@ -6923,7 +7183,13 @@ async fn handle_forward_proxy( .await; } let _ = upstream.shutdown().await; - crate::l7::relay::reject_body_credential(client, *error).await?; + crate::l7::relay::reject_body_credential( + client, + *error, + &l7_ctx, + Some(&correlation), + ) + .await?; return Ok(()); } if let Some(error) = report.downcast_ref::() { @@ -6932,8 +7198,14 @@ async fn handle_forward_proxy( .end(openshell_core::proto::MiddlewareSessionEndReason::Cancellation) .await; } - crate::l7::relay::reject_credential_resolution(client, &l7_ctx, method, error) - .await?; + crate::l7::relay::reject_credential_resolution( + client, + &l7_ctx, + method, + error, + Some(&correlation), + ) + .await?; return Ok(()); } Err(report) @@ -6953,7 +7225,7 @@ async fn handle_forward_proxy( // The request has now survived middleware, token grant, credential // rewriting, generation checks, and the HTTP relay. Only now record the // final allowed outcome. - egress_emit!(build_forward_allow_ocsf_event( + correlation.emit(build_forward_allow_ocsf_event( workload_addr, method, &host_lc, @@ -7103,6 +7375,47 @@ fn build_json_error_response(status: u16, status_text: &str, error: &str, detail .into_bytes() } +fn build_json_error_response_with_request_id( + status: u16, + status_text: &str, + error: &str, + detail: &str, + request_id: Option<&str>, +) -> Vec { + request_id.map_or_else( + || build_json_error_response(status, status_text, error, detail), + |id| build_correlated_json_error_response(status, status_text, error, detail, id), + ) +} + +fn build_correlated_json_error_response( + status: u16, + status_text: &str, + error: &str, + detail: &str, + request_id: &str, +) -> Vec { + let body = serde_json::json!({ + "error": error, + "detail": detail, + "request_id": request_id, + }) + .to_string(); + format!( + "HTTP/1.1 {status} {status_text}\r\n\ + Content-Type: application/json\r\n\ + {}: {request_id}\r\n\ + Content-Length: {}\r\n\ + Connection: close\r\n\ + \r\n\ + {}", + openshell_core::token_grant_failure::REQUEST_ID_HEADER, + body.len(), + body, + ) + .into_bytes() +} + /// Build the forward proxy's refusal for a failed dynamic token grant. /// /// This is the generic grant refusal plus a request ID header, which lets the @@ -7157,6 +7470,7 @@ fn build_json_error_response_with_reason( fn build_middleware_deny_response( policy_name: &str, denial: &openshell_supervisor_middleware::MiddlewareDenial, + request_id: Option<&str>, ) -> Vec { let mut body = serde_json::Map::new(); body.insert("error".to_string(), serde_json::json!("middleware_denied")); @@ -7172,11 +7486,21 @@ fn build_middleware_deny_response( if let Some(reason_code) = &denial.reason_code { body.insert("reason_code".to_string(), serde_json::json!(reason_code)); } + if let Some(request_id) = request_id { + body.insert("request_id".to_string(), serde_json::json!(request_id)); + } let body_str = serde_json::Value::Object(body).to_string(); + let request_id_header = request_id.map_or_else(String::new, |request_id| { + format!( + "{}: {request_id}\r\n", + openshell_core::token_grant_failure::REQUEST_ID_HEADER + ) + }); format!( "HTTP/1.1 403 Forbidden\r\n\ Content-Type: application/json\r\n\ Content-Length: {}\r\n\ + {request_id_header}\ Connection: close\r\n\ \r\n\ {}", @@ -7186,25 +7510,39 @@ fn build_middleware_deny_response( .into_bytes() } -fn build_middleware_failure_response(policy_name: &str) -> Vec { - build_middleware_platform_response(policy_name, "403 Forbidden") +fn build_middleware_failure_response(policy_name: &str, request_id: Option<&str>) -> Vec { + build_middleware_platform_response(policy_name, "403 Forbidden", request_id) } -fn build_middleware_unavailable_response(policy_name: &str) -> Vec { - build_middleware_platform_response(policy_name, "503 Service Unavailable") +fn build_middleware_unavailable_response(policy_name: &str, request_id: Option<&str>) -> Vec { + build_middleware_platform_response(policy_name, "503 Service Unavailable", request_id) } -fn build_middleware_platform_response(policy_name: &str, status: &str) -> Vec { - let body = serde_json::json!({ +fn build_middleware_platform_response( + policy_name: &str, + status: &str, + request_id: Option<&str>, +) -> Vec { + let mut body = serde_json::json!({ "error": "middleware_failed", "detail": "Request could not be processed by configured middleware", "policy": policy_name, }); + if let (Some(request_id), serde_json::Value::Object(fields)) = (request_id, &mut body) { + fields.insert("request_id".to_string(), serde_json::json!(request_id)); + } let body_str = body.to_string(); + let request_id_header = request_id.map_or_else(String::new, |request_id| { + format!( + "{}: {request_id}\r\n", + openshell_core::token_grant_failure::REQUEST_ID_HEADER + ) + }); format!( "HTTP/1.1 {status}\r\n\ Content-Type: application/json\r\n\ Content-Length: {}\r\n\ + {request_id_header}\ Connection: close\r\n\ \r\n\ {}", @@ -8625,7 +8963,7 @@ network_policies: .await .unwrap(); workload.shutdown().await.unwrap(); - let handler = tokio::spawn(handle_mediated_connection( + let mut handler = tokio::spawn(handle_mediated_connection( tokio::io::BufReader::new(stream), supplied_identity, socket_addrs, @@ -8647,10 +8985,11 @@ network_policies: None, None, )); - let request = tokio::time::timeout(std::time::Duration::from_secs(5), request_rx) - .await - .expect("upstream never received the workload's request") - .unwrap(); + let request = tokio::select! { + result = tokio::time::timeout(std::time::Duration::from_secs(5), request_rx) => + result.expect("upstream never received the workload's request").unwrap(), + result = &mut handler => panic!("proxy ended before forwarding the workload request: {result:?}"), + }; assert!( request.starts_with(b"GET /ping HTTP/1.1\r\n"), "upstream received {:?} instead of the workload's request", @@ -10510,7 +10849,7 @@ network_policies: #[test] fn middleware_failure_response_uses_platform_text_without_policy_guidance() { - let response = build_middleware_failure_response("api-policy"); + let response = build_middleware_failure_response("api-policy", None); let response = String::from_utf8(response).expect("UTF-8 error response"); let (_, body) = response.split_once("\r\n\r\n").expect("HTTP response"); let body: serde_json::Value = serde_json::from_str(body).expect("JSON response"); @@ -10528,7 +10867,7 @@ network_policies: #[test] fn middleware_unavailable_response_is_complete_and_has_no_retry_hint() { - let response = build_middleware_unavailable_response("api-policy"); + let response = build_middleware_unavailable_response("api-policy", None); let response = String::from_utf8(response).expect("UTF-8 error response"); assert!(response.starts_with("HTTP/1.1 503 Service Unavailable\r\n")); assert!(!response.to_ascii_lowercase().contains("retry-after")); @@ -15107,6 +15446,7 @@ network_policies: config_name: "prototype-content-guard".into(), reason_code: Some("content_match".into()), }, + None, ); let resp_str = String::from_utf8(resp).unwrap(); let body_start = resp_str.find("\r\n\r\n").unwrap() + 4; diff --git a/crates/openshell-supervisor-network/src/proxy/relay.rs b/crates/openshell-supervisor-network/src/proxy/relay.rs index 495c22241b..f49f0367ca 100644 --- a/crates/openshell-supervisor-network/src/proxy/relay.rs +++ b/crates/openshell-supervisor-network/src/proxy/relay.rs @@ -404,6 +404,65 @@ mod tests { } } + async fn read_http_response(reader: &mut R) -> Vec + where + R: AsyncRead + Unpin, + { + use tokio::io::AsyncReadExt; + + let mut response = Vec::new(); + while !response.ends_with(b"\r\n\r\n") { + response.push(reader.read_u8().await.unwrap()); + assert!(response.len() < 4096); + } + let headers = String::from_utf8_lossy(&response); + let content_length = headers + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().unwrap()) + }) + .unwrap_or_default(); + for _ in 0..content_length { + response.push(reader.read_u8().await.unwrap()); + } + response + } + + fn assert_passthrough_response( + response: &[u8], + expected_body: &[u8], + expect_connection_close: bool, + ) { + let header_end = response + .windows(4) + .position(|bytes| bytes == b"\r\n\r\n") + .expect("response headers") + + 4; + let headers = String::from_utf8_lossy(&response[..header_end]); + assert!(headers.starts_with("HTTP/1.1 200 OK\r\n"), "{headers}"); + assert!( + headers.contains(&format!("Content-Length: {}\r\n", expected_body.len())), + "{headers}" + ); + assert_eq!( + headers + .lines() + .any(|line| line.eq_ignore_ascii_case("Connection: close")), + expect_connection_close, + "{headers}" + ); + assert!( + !headers + .lines() + .any(|line| line.split_once(':').is_some_and(|(name, _)| name + .eq_ignore_ascii_case(openshell_core::token_grant_failure::REQUEST_ID_HEADER))), + "uninspected passthrough must not claim an inspected request receipt" + ); + assert_eq!(&response[header_end..], expected_body); + } + async fn assert_response_lifecycle(mut caller: C, mut client: P) where C: AsyncRead + AsyncWrite + Unpin, @@ -443,16 +502,15 @@ mod tests { let request = b"GET / HTTP/1.1\r\nHost: example.com\r\n\r\n"; caller.write_all(request).await.unwrap(); caller.flush().await.unwrap(); - let mut first = vec![0; persistent.len()]; - caller.read_exact(&mut first).await.unwrap(); - assert_eq!(first, persistent); + let first = read_http_response(&mut caller).await; + assert_passthrough_response(&first, b"hello", false); // A real second exchange verifies reuse, without a timing assertion. caller.write_all(request).await.unwrap(); caller.flush().await.unwrap(); let mut second = Vec::new(); // TLS must return clean EOF, not UnexpectedEof from a dropped socket. caller.read_to_end(&mut second).await.unwrap(); - assert_eq!(second, closing); + assert_passthrough_response(&second, &body, true); }; tokio::time::timeout(std::time::Duration::from_secs(5), async { let (result, (), ()) = tokio::join!(relay, serve, receive); diff --git a/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs b/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs index 3623b1dbb7..68e53c27a3 100644 --- a/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs +++ b/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs @@ -37,6 +37,7 @@ fn assert_json_response( expected_status: &str, expected_error: &str, expected_detail: &str, + expected_request_id: Option<&str>, ) { let (headers, body) = response .windows(4) @@ -48,12 +49,19 @@ fn assert_json_response( assert!(headers.contains("Content-Type: application/json\r\n")); assert!(headers.contains(&format!("Content-Length: {}\r\n", body.len()))); assert!(headers.contains("Connection: close\r\n")); + if let Some(request_id) = expected_request_id { + assert!(headers.contains(&format!("x-openshell-request-id: {request_id}\r\n"))); + } + let mut expected = serde_json::json!({ + "error": expected_error, + "detail": expected_detail, + }); + if let Some(request_id) = expected_request_id { + expected["request_id"] = serde_json::json!(request_id); + } assert_eq!( serde_json::from_slice::(body).unwrap(), - serde_json::json!({ - "error": expected_error, - "detail": expected_detail, - }) + expected ); } @@ -111,9 +119,26 @@ async fn destination_denials_preserve_adapter_specific_wire_contracts() { "HTTP/1.1 403 Forbidden\r\n", "ssrf_denied", &format!("CONNECT target.example:8443 blocked: {detail}"), + None, ); let (mut app, mut proxy) = tcp_pair().await; + let request = crate::l7::provider::L7Request { + action: "POST".to_string(), + target: "/v1/items".to_string(), + query_params: std::collections::HashMap::new(), + raw_header: b"POST /v1/items HTTP/1.1\r\nHost: target.example:8080\r\n\r\n".to_vec(), + body_length: crate::l7::provider::BodyLength::None, + }; + let correlation = crate::l7::correlation::RequestCorrelation::with_request_id( + &request, + &crate::l7::relay::L7EvalContext { + host: "target.example".to_string(), + port: 8080, + ..Default::default() + }, + "test-request-id".to_string(), + ); deny_forward_destination( &mut proxy, &tracing::Span::none(), @@ -134,6 +159,7 @@ async fn destination_denials_preserve_adapter_specific_wire_contracts() { )), None, None, + Some(&correlation), ) .await .unwrap(); @@ -145,6 +171,7 @@ async fn destination_denials_preserve_adapter_specific_wire_contracts() { "HTTP/1.1 403 Forbidden\r\n", "ssrf_denied", &format!("POST target.example:8080 blocked: {detail}"), + Some("test-request-id"), ); } } diff --git a/crates/openshell-supervisor-network/src/proxy/trace_tests.rs b/crates/openshell-supervisor-network/src/proxy/trace_tests.rs index 1602913885..d6871ca941 100644 --- a/crates/openshell-supervisor-network/src/proxy/trace_tests.rs +++ b/crates/openshell-supervisor-network/src/proxy/trace_tests.rs @@ -16,6 +16,7 @@ async fn paired_credential_denial_shares_one_deny_span() { 443, "credential-policy", &tracing::Span::none(), + None, ); }) .await; @@ -38,6 +39,7 @@ async fn repeated_credential_denials_keep_decision_spans_separate() { 443, "credential-policy", &connect, + None, ); } let allowed = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -133,6 +135,7 @@ async fn forward_denials_use_the_saved_connect_parent() { &decision, None, None, + None, )) .unwrap(); assert!(output.starts_with(b"HTTP/1.1 403 Forbidden")); @@ -194,7 +197,7 @@ async fn forward_evaluation_error_keeps_the_http_decision_policy_context() { "evaluation unavailable", ); // Use the same grouping as the forward-proxy production path. - emit_forward_l7_decision(decision, Some(error), &connect); + emit_forward_l7_decision(decision, Some(error), &connect, None); }) .await; diff --git a/crates/openshell-supervisor-network/src/telemetry.rs b/crates/openshell-supervisor-network/src/telemetry.rs index 73e573b985..84c48dbbae 100644 --- a/crates/openshell-supervisor-network/src/telemetry.rs +++ b/crates/openshell-supervisor-network/src/telemetry.rs @@ -17,6 +17,12 @@ macro_rules! egress_emit { } pub(crate) use egress_emit; +/// Emit an inspected request event whose request already owns the decision span. +#[allow(clippy::disallowed_macros)] +pub fn emit_correlated(event: OcsfEvent) { + openshell_ocsf::ocsf_emit!(event); +} + fn is_denied(event: &OcsfEvent) -> bool { match event { OcsfEvent::NetworkActivity(event) => event.action == Some(ActionId::Denied), diff --git a/docs/observability/ocsf-json-export.mdx b/docs/observability/ocsf-json-export.mdx index 2f4b52c10f..30e5fb1187 100644 --- a/docs/observability/ocsf-json-export.mdx +++ b/docs/observability/ocsf-json-export.mdx @@ -311,12 +311,32 @@ HTTP Activity and API Activity records emitted inside a sampled OpenTelemetry sp OTLP log export links every OCSF record, whatever its class, to the active sampled span through the log record's own trace and span ID fields. Use those fields for log-to-trace links in your backend. -With OTLP trace export enabled on a dedicated supervisor, denied HTTP requests can create an INFO `supervisor.egress.deny` span around the audit records for that decision. The span records the policy, reason, and destination when available. At the default log level, every deny span is a root span. At debug level, FORWARD policy, destination, credential-binding, and request middleware denials can join the available connect trace and its authorization, resolve, and dial spans. Other deny spans, including L7 denials inside CONNECT tunnels and response middleware denials, remain root spans. Shared supervisor hosts do not initialize OTLP exporters, so their records carry no trace correlation. Workload `traceparent` headers never supply supervisor correlation. +With OTLP trace export enabled on a dedicated supervisor, HTTP refusals outside the inspected-request path can create an INFO `supervisor.egress.deny` span around the audit records for that decision. The span records the policy, reason, and destination when available. At the default log level, these are root spans. At debug level, FORWARD denials can join the available connect trace. Inspected requests use the request and policy spans described below instead. Shared supervisor hosts do not initialize OTLP exporters. Uninspected traffic does not adopt workload trace headers. To bound telemetry volume when a workload retries blocked destinations, the supervisor creates at most 10 deny spans per destination at once and one more per second after that. A process-wide budget also allows at most 100 spans at once and refills at 100 per second across destinations. Rate-limited decisions create no additional deny span, while all audit records are still emitted. Records can retain correlation to an already active sampled span, including a connect span at debug level. Gateway records and their OTLP trace context can inherit the caller's incoming `traceparent`, including before authentication succeeds. The caller also decides whether they are correlated at all, because an unsampled `traceparent` suppresses `trace.uid`. Treat gateway trace IDs as correlation hints, not evidence of the caller's identity, and do not treat a missing `trace.uid` as meaningful. +## Join Inspected HTTP Requests + +Inspected HTTP requests create an `openshell.http.request` span and an +`openshell.policy.evaluate` child when policy evaluation runs. One valid, +sampled workload `traceparent` supplies an untrusted parent. Invalid or duplicate +headers are ignored. Unsampled context is linked rather than adopted, and +`tracestate` is not adopted. + +The supervisor assigns a request UUID shared by middleware, the request span, +OCSF `metadata.correlation_uid`, and the `x-openshell-request-id` response header. +It removes workload, middleware, and upstream attempts to set that header on +inspected paths. Neither the trace ID nor the receipt proves identity or +authorization. L4 and uninspected traffic do not provide this receipt guarantee. + +One primary HTTP Activity record is emitted when the final response head is +written. If the request ends earlier, a guard emits the available outcome. +The receipt does not prove that the body was delivered. Records still join +through the request UUID when no sampled trace exists. Security logs do not +need to be returned to the sandbox. + ## OCSF Event Classes in JSON The `class_uid` field identifies the event type: