From 3776371dcec51b0828629d95d334fad0ead6b7a4 Mon Sep 17 00:00:00 2001 From: Christian Date: Tue, 10 Nov 2026 20:28:29 -0600 Subject: [PATCH 1/3] Extract explicit EC storage selection and celld persistence --- Cargo.lock | 11 + Cargo.toml | 1 + crates/trusted-server-adapter-axum/Cargo.toml | 4 + crates/trusted-server-adapter-axum/src/app.rs | 3 + .../src/celld_ec.rs | 120 ++++++ .../trusted-server-adapter-axum/src/ec_kv.rs | 94 ++--- .../src/ec_kv_tests.rs | 68 +++- crates/trusted-server-adapter-axum/src/lib.rs | 2 + .../src/startup.rs | 61 ++- .../tests/ec_kv_configuration.rs | 140 +++---- .../tests/native_acceptance.rs | 173 ++++++++ .../tests/native_ec.rs | 126 +++++- .../tests/startup.rs | 96 +++++ .../tests/support/celld.rs | 105 +++++ .../src/app.rs | 29 ++ .../trusted-server-adapter-fastly/src/app.rs | 11 +- .../trusted-server-adapter-fastly/src/main.rs | 58 ++- crates/trusted-server-adapter-spin/src/app.rs | 29 ++ crates/trusted-server-core/Cargo.toml | 1 + crates/trusted-server-core/src/ec/mod.rs | 1 + crates/trusted-server-core/src/ec/storage.rs | 223 +++++++++++ crates/trusted-server-core/src/settings.rs | 5 + .../tests/ec_storage_config.rs | 190 +++++++++ .../configs/trusted-server.integration.toml | 4 +- .../tests/common/config.rs | 37 +- docs/guide/native-ec-persistence.md | 66 ++- ...c-storage-provider-configuration-design.md | 377 ++++++++++++++++++ scripts/test-native-ec-image.mjs | 12 +- trusted-server.example.toml | 11 +- 29 files changed, 1871 insertions(+), 187 deletions(-) create mode 100644 crates/trusted-server-adapter-axum/src/celld_ec.rs create mode 100644 crates/trusted-server-adapter-axum/tests/support/celld.rs create mode 100644 crates/trusted-server-core/src/ec/storage.rs create mode 100644 crates/trusted-server-core/tests/ec_storage_config.rs create mode 100644 docs/superpowers/specs/2026-11-03-ec-storage-provider-configuration-design.md diff --git a/Cargo.lock b/Cargo.lock index 0ee33ae6e6..bf6401adaa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2233,8 +2233,17 @@ name = "edgezero-store-celld" version = "0.1.0" source = "git+https://github.com/IABTechLab/edgezero.git?rev=83d7a6296d285e135c4ea70021b19e6c8dca7e9f#83d7a6296d285e135c4ea70021b19e6c8dca7e9f" dependencies = [ + "async-trait", + "base64", + "bytes", + "edgezero-core", + "futures-util", + "reqwest 0.13.4", "serde", + "serde_json", + "sha2 0.11.1", "thiserror 2.0.18", + "tokio", "url", ] @@ -7120,6 +7129,7 @@ dependencies = [ "base64", "edgezero-adapter-axum", "edgezero-core", + "edgezero-store-celld", "error-stack", "futures", "log", @@ -7276,6 +7286,7 @@ dependencies = [ "derive_more", "ed25519-dalek", "edgezero-core", + "edgezero-store-celld", "error-stack", "flate2", "futures", diff --git a/Cargo.toml b/Cargo.toml index e26504962a..a058c44120 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,7 @@ edgezero-adapter-fastly = { git = "https://github.com/IABTechLab/edgezero.git", edgezero-adapter-spin = { git = "https://github.com/IABTechLab/edgezero.git", rev = "83d7a6296d285e135c4ea70021b19e6c8dca7e9f", default-features = false } edgezero-cli = { git = "https://github.com/IABTechLab/edgezero.git", rev = "83d7a6296d285e135c4ea70021b19e6c8dca7e9f" } edgezero-core = { git = "https://github.com/IABTechLab/edgezero.git", rev = "83d7a6296d285e135c4ea70021b19e6c8dca7e9f", default-features = false } +edgezero-store-celld = { git = "https://github.com/IABTechLab/edgezero.git", rev = "83d7a6296d285e135c4ea70021b19e6c8dca7e9f", default-features = false } env_logger = "0.11" error-stack = "0.6" esi = "0.7.2" diff --git a/crates/trusted-server-adapter-axum/Cargo.toml b/crates/trusted-server-adapter-axum/Cargo.toml index d31db35391..0b5ddb2e7e 100644 --- a/crates/trusted-server-adapter-axum/Cargo.toml +++ b/crates/trusted-server-adapter-axum/Cargo.toml @@ -24,6 +24,7 @@ aws-appconfig-agent = ["edgezero-adapter-axum/aws-appconfig-agent"] aws-secrets-manager = ["edgezero-adapter-axum/aws-secrets-manager"] aws-dynamodb-ec = ["dep:aws-config", "dep:aws-sdk-dynamodb"] geo-cloudfront = ["edgezero-adapter-axum/geo-cloudfront"] +celld-ec = ["dep:edgezero-store-celld"] [dependencies] async-trait = { workspace = true } @@ -41,6 +42,9 @@ simple_logger = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "sync", "time"] } trusted-server-core = { workspace = true } +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +edgezero-store-celld = { workspace = true, features = ["records"], optional = true } + [dev-dependencies] serde_json = { workspace = true } tempfile = { workspace = true } diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index e45f7807f7..244d29f520 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -83,6 +83,9 @@ fn build_state_with_services( settings: Settings, services: Option, ) -> Result, Report> { + settings + .ec + .resolve_storage(trusted_server_core::ec::storage::EcRuntime::Native)?; let plan = Arc::new(compile_auction_plan(&settings)?); plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Axum)?; let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; diff --git a/crates/trusted-server-adapter-axum/src/celld_ec.rs b/crates/trusted-server-adapter-axum/src/celld_ec.rs new file mode 100644 index 0000000000..df677627cc --- /dev/null +++ b/crates/trusted-server-adapter-axum/src/celld_ec.rs @@ -0,0 +1,120 @@ +//! Native EC persistence through celld's typed conditional-record client. + +use async_trait::async_trait; +use edgezero_core::EdgeError; +use edgezero_core::time::MonotonicClock; +use edgezero_store_celld::CelldPreparation; +use edgezero_store_celld::records::{ + CelldRecordStore, RecordWrite, RecordWriteMode, RecordWriteOutcome, +}; +use edgezero_store_celld::settings::{CelldAccessToken, CelldKvSettings, CelldPreparationLimits}; +use error_stack::Report; +use trusted_server_core::ec::kv_backend::{ + EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteMode, EcKvWriteOutcome, +}; +use trusted_server_core::error::TrustedServerError; + +/// A live provider. Clones share transport, never cached identity rows. +#[derive(Clone, Debug)] +pub struct CelldEcKv { + records: CelldRecordStore, +} + +fn preparation_error() -> EdgeError { + EdgeError::internal(std::io::Error::other("native EC celld preparation failed")) +} + +fn store_error() -> Report { + Report::new(TrustedServerError::KvStore { + store_name: "native-ec-celld".to_owned(), + message: "EC celld operation failed".to_owned(), + }) +} + +impl CelldEcKv { + /// Prepare an explicitly selected provider using one captured bootstrap token. + /// + /// # Errors + /// Invalid settings, credentials, protocol or service failures prevent readiness. + pub async fn prepare( + settings: &CelldKvSettings, + token: Option<&CelldAccessToken>, + ) -> Result { + let mut preparation = + CelldPreparation::new(MonotonicClock::default(), CelldPreparationLimits::default()) + .map_err(|_| preparation_error())?; + let records = preparation + .prepare_records(settings, token) + .await + .map_err(|_| preparation_error())?; + Ok(Self { records }) + } +} + +#[async_trait(?Send)] +impl EcKvStore for CelldEcKv { + fn store_name(&self) -> &str { + "native-ec-celld" + } + async fn lookup(&self, key: &str) -> Result, Report> { + self.records + .lookup(key) + .await + .map(|row| { + row.map(|row| EcKvLookup { + body: row.body, + metadata: row.metadata, + generation: row.generation, + }) + }) + .map_err(|_| store_error()) + } + async fn key_exists(&self, key: &str) -> Result> { + self.records + .key_exists(key) + .await + .map_err(|_| store_error()) + } + async fn insert( + &self, + key: &str, + write: EcKvWrite<'_>, + ) -> Result> { + let mode = match write.mode { + EcKvWriteMode::Add => RecordWriteMode::Add, + EcKvWriteMode::Overwrite => RecordWriteMode::Overwrite, + EcKvWriteMode::IfGenerationMatch(generation) => { + RecordWriteMode::IfGenerationMatch(generation) + } + }; + self.records + .insert( + key, + RecordWrite { + body: write.body.as_bytes(), + metadata: Some(write.metadata.as_bytes()), + ttl: write.ttl, + mode, + }, + ) + .await + .map(|outcome| match outcome { + RecordWriteOutcome::Written => EcKvWriteOutcome::Written, + RecordWriteOutcome::PreconditionFailed => EcKvWriteOutcome::PreconditionFailed, + }) + .map_err(|_| store_error()) + } + async fn list_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result, Report> { + self.records + .list_keys_with_prefix(prefix, limit) + .await + .map_err(|_| store_error()) + } + async fn delete(&self, key: &str) -> Result<(), Report> { + self.records.delete(key).await.map_err(|_| store_error()) + } +} diff --git a/crates/trusted-server-adapter-axum/src/ec_kv.rs b/crates/trusted-server-adapter-axum/src/ec_kv.rs index a71548370e..0d2a4412b0 100644 --- a/crates/trusted-server-adapter-axum/src/ec_kv.rs +++ b/crates/trusted-server-adapter-axum/src/ec_kv.rs @@ -12,6 +12,7 @@ use async_trait::async_trait; use edgezero_core::EdgeError; use error_stack::Report; use trusted_server_core::ec::kv_backend::{EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteOutcome}; +use trusted_server_core::ec::storage::DynamoDbEcStorage; use trusted_server_core::error::TrustedServerError; /// Cloneable live provider. Clones share the SDK connection pool, not a cache. @@ -40,29 +41,32 @@ fn store_error() -> Report { }) } -/// Prepare the explicitly selected EC backend before accepting requests. -/// -/// `TS_EC_DYNAMODB_ENABLED=true` selects this backend. Absent or `false` leaves -/// it off, regardless of AWS configuration. Selection requires REGION, TABLE, -/// and NAMESPACE under the same prefix, and the `aws-dynamodb-ec` build feature. -/// Credentials use the normal AWS SDK chain. For isolated local tests only, -/// `TS_EC_DYNAMODB_LOCAL_ENDPOINT` accepts an HTTP root URL at `127.0.0.1` with -/// an explicit port and bypasses the credential chain with dummy credentials. -/// Provider errors never include SDK responses, identifiers, endpoint URLs, -/// credentials, or record contents. +/// Reject obsolete native launch selection without preparing from environment. /// /// # Errors -/// Returns a sanitized error on invalid selection or failed preparation. -pub async fn prepare_from_env() -> Result>, EdgeError> { +/// Enabled or malformed legacy selection must be migrated to published config. +pub fn reject_legacy_configuration() -> Result<(), EdgeError> { match std::env::var("TS_EC_DYNAMODB_ENABLED") { - Err(std::env::VarError::NotPresent) => return Ok(None), - Ok(value) if value == "false" => return Ok(None), - Ok(value) if value == "true" => {} - _ => return Err(preparation_error()), + Err(std::env::VarError::NotPresent) => Ok(()), + Ok(value) if value == "false" => Ok(()), + Ok(value) if value == "true" => Err(EdgeError::internal(std::io::Error::other( + "TS_EC_DYNAMODB_ENABLED is obsolete; publish ec.storage and remove legacy EC environment selection", + ))), + _ => Err(EdgeError::internal(std::io::Error::other( + "TS_EC_DYNAMODB_ENABLED is malformed; migrate to published ec.storage configuration", + ))), } +} + +/// Prepare the explicitly selected typed `DynamoDB` provider before readiness. +/// +/// # Errors +/// Returns a sanitized error on invalid input, missing feature or failed preparation. +pub async fn prepare(config: &DynamoDbEcStorage) -> Result, EdgeError> { + config.validate().map_err(|_| preparation_error())?; #[cfg(feature = "aws-dynamodb-ec")] { - enabled::prepare_from_env().await.map(Some) + enabled::prepare(config).await } #[cfg(not(feature = "aws-dynamodb-ec"))] Err(EdgeError::internal(std::io::Error::other( @@ -126,26 +130,16 @@ mod enabled { type Item = HashMap; type StoreResult = Result>; - pub(super) async fn prepare_from_env() -> Result, EdgeError> { - let required = |suffix: &str| { - std::env::var(format!("TS_EC_DYNAMODB_{suffix}")).map_err(|_| preparation_error()) - }; - let region = required("REGION")?; - let table = required("TABLE")?; - let namespace = required("NAMESPACE")?; - if region.is_empty() || region.chars().any(char::is_whitespace) { - return Err(preparation_error()); - } - validate_binding(&table, &namespace)?; + pub(super) async fn prepare( + config: &super::DynamoDbEcStorage, + ) -> Result, EdgeError> { tokio::time::timeout(PREPARATION_TIMEOUT, async { - let builder = match std::env::var("TS_EC_DYNAMODB_LOCAL_ENDPOINT") { - Ok(endpoint) => { - validate_local_endpoint(&endpoint)?; - // Explicit emulator mode never reads local AWS files or - // sends real workload credentials to the test endpoint. + let builder = match &config.local_endpoint { + Some(endpoint) => { + // Explicit emulator mode never reads ambient credentials. aws_sdk_dynamodb::Config::builder() .behavior_version(BehaviorVersion::v2026_01_12()) - .region(Region::new(region)) + .region(Region::new(config.region.clone())) .credentials_provider(aws_sdk_dynamodb::config::Credentials::new( "local", "local", @@ -155,27 +149,25 @@ mod enabled { )) .endpoint_url(endpoint) } - Err(std::env::VarError::NotPresent) => { + None => { let shared = aws_config::defaults(BehaviorVersion::v2026_01_12()) - .region(Region::new(region)) + .region(Region::new(config.region.clone())) .load() .await; let mut builder = aws_sdk_dynamodb::Config::from(&shared).to_builder(); - // Clear both global and service-specific endpoint URLs - // copied from the shared config. Resolve normal AWS endpoints. + // Production ignores ambient endpoint overrides. builder.set_endpoint_url(None); builder } - Err(_) => return Err(preparation_error()), }; - let config = builder + let config_sdk = builder .retry_config(RetryConfig::standard().with_max_attempts(1)) .timeout_config(timeout_config()) .build(); DynamoDbEcKv::prepare_with_client( - aws_sdk_dynamodb::Client::from_conf(config), - table, - namespace, + aws_sdk_dynamodb::Client::from_conf(config_sdk), + config.table.clone(), + config.namespace.clone(), ) .await }) @@ -183,22 +175,6 @@ mod enabled { .map_err(|_| preparation_error())? } - pub(super) fn validate_local_endpoint(endpoint: &str) -> Result<(), EdgeError> { - let url = reqwest::Url::parse(endpoint).map_err(|_| preparation_error())?; - if url.scheme() != "http" - || url.host_str() != Some("127.0.0.1") - || url.port().is_none() - || !url.username().is_empty() - || url.password().is_some() - || url.path() != "/" - || url.query().is_some() - || url.fragment().is_some() - { - return Err(preparation_error()); - } - Ok(()) - } - fn validate_binding(table: &str, namespace: &str) -> Result<(), EdgeError> { if !(3..=255).contains(&table.len()) || !table diff --git a/crates/trusted-server-adapter-axum/src/ec_kv_tests.rs b/crates/trusted-server-adapter-axum/src/ec_kv_tests.rs index abccc1016f..45defdda53 100644 --- a/crates/trusted-server-adapter-axum/src/ec_kv_tests.rs +++ b/crates/trusted-server-adapter-axum/src/ec_kv_tests.rs @@ -70,6 +70,58 @@ fn table_description() -> Value { }}) } +#[tokio::test(flavor = "current_thread")] +async fn typed_local_preparation_uses_envelope_binding_and_dummy_credentials() { + let captured = Arc::new(Mutex::new(Vec::::new())); + let headers = Arc::clone(&captured); + let router = axum::Router::new().fallback(move |request: Request| { + headers + .lock() + .expect("should capture typed preparation request") + .push(request.headers().clone()); + async { axum::Json(table_description()) } + }); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("should bind typed preparation fixture"); + let endpoint = format!( + "http://{}", + listener + .local_addr() + .expect("should read local fixture address") + ); + let task = tokio::spawn(async move { + axum::serve(listener, router) + .await + .expect("should serve typed preparation fixture"); + }); + let config = trusted_server_core::ec::storage::DynamoDbEcStorage { + region: "us-east-1".to_owned(), + table: TABLE.to_owned(), + namespace: " exact ".to_owned(), + local_endpoint: Some(endpoint), + }; + let provider = super::prepare(&config) + .await + .expect("should prepare from typed envelope fields"); + assert_eq!( + provider.partition, "ec: exact ", + "should preserve exact namespace bytes" + ); + let headers = captured.lock().expect("should inspect preparation headers"); + assert_eq!(headers.len(), 1, "should not retry preparation"); + let authorization = headers[0] + .get("authorization") + .expect("should sign local request") + .to_str() + .expect("should decode local authorization"); + assert!( + authorization.contains("Credential=local/"), + "should bypass ambient credentials with explicit local dummy credentials" + ); + task.abort(); +} + async fn fixture() -> Fixture { let requests: Requests = Arc::new(Mutex::new(Vec::new())); let responses: Responses = Arc::new(Mutex::new(VecDeque::from([( @@ -401,14 +453,20 @@ async fn malformed_cursor_expiry_or_allocator_fails_closed() { } /// Run explicitly against disposable `DynamoDB` Local, never AWS: -/// `TS_EC_DYNAMODB_LOCAL_ENDPOINT=http://127.0.0.1:PORT cargo test ... -- --ignored` +/// `TS_TEST_EC_DYNAMODB_ENDPOINT=http://127.0.0.1:PORT cargo test ... -- --ignored` #[tokio::test(flavor = "current_thread")] #[ignore = "requires explicitly selected disposable DynamoDB Local loopback endpoint"] async fn dynamodb_local_contract() { - let endpoint = std::env::var("TS_EC_DYNAMODB_LOCAL_ENDPOINT") - .expect("should explicitly select a Local endpoint"); - super::enabled::validate_local_endpoint(&endpoint) - .expect("should restrict DynamoDB Local tests to a loopback root URL"); + let endpoint = std::env::var("TS_TEST_EC_DYNAMODB_ENDPOINT") + .expect("should explicitly select a test-only Local endpoint"); + trusted_server_core::ec::storage::DynamoDbEcStorage { + region: "us-east-1".to_owned(), + table: "example-identity".to_owned(), + namespace: "fixture".to_owned(), + local_endpoint: Some(endpoint.clone()), + } + .validate() + .expect("should restrict DynamoDB Local tests to a loopback root URL"); let client = client(&endpoint); let table = format!( "ec-contract-{}-{}", diff --git a/crates/trusted-server-adapter-axum/src/lib.rs b/crates/trusted-server-adapter-axum/src/lib.rs index 69160fc62f..533093138e 100644 --- a/crates/trusted-server-adapter-axum/src/lib.rs +++ b/crates/trusted-server-adapter-axum/src/lib.rs @@ -6,6 +6,8 @@ /// Application routing and handler registration for the Axum dev server. pub mod app; +#[cfg(all(feature = "celld-ec", not(target_arch = "wasm32")))] +pub mod celld_ec; /// Native live DynamoDB conditional identity storage and provider selection. pub mod ec_kv; /// Request middleware (auth, response finalisation). diff --git a/crates/trusted-server-adapter-axum/src/startup.rs b/crates/trusted-server-adapter-axum/src/startup.rs index 9eed039d5d..d87f13962c 100644 --- a/crates/trusted-server-adapter-axum/src/startup.rs +++ b/crates/trusted-server-adapter-axum/src/startup.rs @@ -9,10 +9,13 @@ use std::sync::Arc; use edgezero_adapter_axum::dev_server::PreparedStores; use edgezero_core::app::App; use edgezero_core::error::EdgeError; +#[cfg(feature = "celld-ec")] +use edgezero_store_celld::settings::CelldAccessToken; use error_stack::Report; use trusted_server_core::config_payload::{ DEFAULT_CONFIG_STORE_ID, DEFAULT_SECRET_STORE_ID, VerifiedConfigBlob, }; +use trusted_server_core::ec::storage::{EcRuntime, EcStorageSelection}; use trusted_server_core::integrations::datadome::{DataDomeConfig, ProtectionMatcherConfig}; use trusted_server_core::platform::{ PlatformConfigStore, PlatformError, PlatformSecretStore, RuntimeServices, StoreId, StoreName, @@ -260,13 +263,63 @@ pub async fn initialize_native(app: &mut App, stores: &PreparedStores) -> Result ) .map_err(settings_error)?; validate_snapshot_consumers(&settings)?; + crate::ec_kv::reject_legacy_configuration()?; let mut state = build_state_with_settings(settings).map_err(settings_error)?; // EC records stay live. Only the validated provider handle is shared, never // mutable identity rows in the immutable config/secret snapshot. - if let Some(provider) = crate::ec_kv::prepare_from_env().await? { - Arc::get_mut(&mut state) - .expect("should uniquely own initializing app state") - .set_ec_store(provider.as_ref().clone()); + let provider_settings = Arc::clone(&state.settings); + match provider_settings + .ec + .resolve_storage(EcRuntime::Native) + .map_err(settings_error)? + { + None => {} + Some(EcStorageSelection::DynamoDb(config)) => { + let provider = crate::ec_kv::prepare(config).await?; + Arc::get_mut(&mut state) + .expect("should uniquely own initializing app state") + .set_ec_store(provider.as_ref().clone()); + } + Some(EcStorageSelection::Celld(config)) => { + #[cfg(feature = "celld-ec")] + { + // Capture only the selected bootstrap reference once. The live + // handle owns this credential; request paths never read env. + let token = config + .access + .token_env() + .map(|name| { + std::env::var(name) + .map_err(|_| { + configuration_error( + "selected EC celld bootstrap token is unavailable", + ) + }) + .and_then(|value| { + CelldAccessToken::new(value).map_err(|_| { + configuration_error( + "selected EC celld bootstrap token is invalid", + ) + }) + }) + }) + .transpose()?; + let provider = crate::celld_ec::CelldEcKv::prepare(config, token.as_ref()).await?; + Arc::get_mut(&mut state) + .expect("should uniquely own initializing app state") + .set_ec_store(provider); + } + #[cfg(not(feature = "celld-ec"))] + { + let _ = config; + return Err(configuration_error( + "native EC celld selected but celld-ec build feature is unavailable", + )); + } + } + Some(EcStorageSelection::Fastly(_)) => { + unreachable!("native resolution should not select Fastly") + } } if deadline.is_expired_at(clock.now()) { return Err(configuration_error( diff --git a/crates/trusted-server-adapter-axum/tests/ec_kv_configuration.rs b/crates/trusted-server-adapter-axum/tests/ec_kv_configuration.rs index fbb5c229d2..52cce60459 100644 --- a/crates/trusted-server-adapter-axum/tests/ec_kv_configuration.rs +++ b/crates/trusted-server-adapter-axum/tests/ec_kv_configuration.rs @@ -1,105 +1,81 @@ -//! Exercise real environment selection in fresh processes, never mutate the -//! parent test runner's environment or consult ambient AWS configuration. +//! The obsolete launch contract is rejected without reading AWS binding values. use std::process::Command; -use trusted_server_adapter_axum::ec_kv::prepare_from_env; +#[cfg(not(feature = "aws-dynamodb-ec"))] +use trusted_server_adapter_axum::ec_kv::prepare; +use trusted_server_adapter_axum::ec_kv::reject_legacy_configuration; +#[cfg(not(feature = "aws-dynamodb-ec"))] +use trusted_server_core::ec::storage::DynamoDbEcStorage; #[test] -fn ec_kv_configuration_is_process_isolated() { - let mut cases = vec!["absent", "false", "empty", "uppercase", "numeric"]; - #[cfg(not(feature = "aws-dynamodb-ec"))] - cases.push("missing-feature"); - #[cfg(feature = "aws-dynamodb-ec")] - cases.extend([ - "missing-binding", - "remote-endpoint", - "https-endpoint", - "endpoint-userinfo", - "endpoint-path", - "endpoint-query", - "endpoint-fragment", - "endpoint-no-port", - ]); - for case in cases { - let output = - Command::new(std::env::current_exe().expect("should locate config test executable")) - .args(["--exact", "ec_kv_configuration_subprocess", "--nocapture"]) - .env_clear() - .env("TS_EC_CONFIGURATION_CASE", case) - .env("AWS_EC2_METADATA_DISABLED", "true") - .envs(configuration_bindings(case)) - .output() - .expect("should execute isolated provider configuration test"); +fn legacy_ec_selection_is_rejected_in_isolated_processes() { + for case in ["absent", "false", "true", "empty", "uppercase", "numeric"] { + let mut command = + Command::new(std::env::current_exe().expect("should locate config test executable")); + command + .args(["--exact", "legacy_ec_selection_subprocess", "--nocapture"]) + .env_clear() + .env("TS_EC_CONFIGURATION_CASE", case) + .env("TS_EC_DYNAMODB_TABLE", "orphaned-unused-table") + .env("TS_EC_DYNAMODB_NAMESPACE", "orphaned-unused-namespace") + .env("TS_EC_DYNAMODB_LOCAL_ENDPOINT", "invalid-secret-endpoint"); + if case != "absent" { + command.env( + "TS_EC_DYNAMODB_ENABLED", + match case { + "empty" => "", + "uppercase" => "TRUE", + "numeric" => "1", + value => value, + }, + ); + } + let output = command + .output() + .expect("should execute isolated migration check"); assert!( output.status.success(), - "configuration case {case} failed: {}", + "should pass case {case}: {}", String::from_utf8_lossy(&output.stderr) ); } } -fn configuration_bindings(case: &str) -> Vec<(&'static str, &'static str)> { - let mut vars = vec![ - ("TS_EC_DYNAMODB_TABLE", "ec-configuration"), - ("TS_EC_DYNAMODB_NAMESPACE", "configuration"), - ("TS_EC_DYNAMODB_REGION", "us-east-1"), - ("TS_EC_DYNAMODB_LOCAL_ENDPOINT", "invalid-unused-endpoint"), - ]; - match case { - "absent" => {} - "false" => vars.push(("TS_EC_DYNAMODB_ENABLED", "false")), - "empty" => vars.push(("TS_EC_DYNAMODB_ENABLED", "")), - "uppercase" => vars.push(("TS_EC_DYNAMODB_ENABLED", "TRUE")), - "numeric" => vars.push(("TS_EC_DYNAMODB_ENABLED", "1")), - "missing-feature" | "missing-binding" => vars.push(("TS_EC_DYNAMODB_ENABLED", "true")), - endpoint_case => { - let endpoint = match endpoint_case { - "remote-endpoint" => "http://example.com:8000", - "https-endpoint" => "https://127.0.0.1:8000", - "endpoint-userinfo" => "http://user:secret@127.0.0.1:8000", - "endpoint-path" => "http://127.0.0.1:8000/private", - "endpoint-query" => "http://127.0.0.1:8000/?secret=value", - "endpoint-fragment" => "http://127.0.0.1:8000/#secret", - "endpoint-no-port" => "http://127.0.0.1", - _ => unreachable!("unknown subprocess fixture case"), - }; - vars.retain(|(name, _)| *name != "TS_EC_DYNAMODB_LOCAL_ENDPOINT"); - vars.push(("TS_EC_DYNAMODB_ENABLED", "true")); - vars.push(("TS_EC_DYNAMODB_LOCAL_ENDPOINT", endpoint)); - } - } - if case == "missing-binding" { - vars.retain(|(name, _)| *name != "TS_EC_DYNAMODB_TABLE"); - } - vars -} - #[test] -fn ec_kv_configuration_subprocess() { +fn legacy_ec_selection_subprocess() { let Ok(case) = std::env::var("TS_EC_CONFIGURATION_CASE") else { return; }; - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .expect("should build isolated configuration runtime"); - let result = runtime.block_on(prepare_from_env()); + let result = reject_legacy_configuration(); if matches!(case.as_str(), "absent" | "false") { + result.expect("should ignore inactive legacy selection and orphaned bindings"); + } else { + let detail = format!( + "{:?}", + result.expect_err("should reject obsolete or malformed selection") + ); assert!( - result - .expect("unselected backend should remain off") - .is_none() + detail.contains("ec.storage"), + "should explain publication migration" ); - } else { - let error = result.expect_err("selected invalid backend should fail closed"); - let detail = format!("{error:?}"); - if case == "missing-feature" { - assert!(detail.contains("aws-dynamodb-ec build feature is unavailable")); - } assert!( - !detail.contains("secret"), - "provider configuration error should not echo input" + !detail.contains("invalid-secret-endpoint"), + "should not echo legacy input" ); } } + +#[cfg(not(feature = "aws-dynamodb-ec"))] +#[tokio::test] +async fn typed_dynamodb_selection_fails_without_compiled_provider() { + let error = prepare(&DynamoDbEcStorage { + region: "us-east-1".to_owned(), + table: "example-identity".to_owned(), + namespace: "fixture".to_owned(), + local_endpoint: None, + }) + .await + .expect_err("should reject selected provider before network access"); + assert!(format!("{error:?}").contains("aws-dynamodb-ec build feature is unavailable")); +} diff --git a/crates/trusted-server-adapter-axum/tests/native_acceptance.rs b/crates/trusted-server-adapter-axum/tests/native_acceptance.rs index 8f39717749..015980278e 100644 --- a/crates/trusted-server-adapter-axum/tests/native_acceptance.rs +++ b/crates/trusted-server-adapter-axum/tests/native_acceptance.rs @@ -23,10 +23,172 @@ use trusted_server_core::config_payload::{ }; use trusted_server_core::test_support::nextjs_auction; +#[cfg(all(feature = "celld-ec", feature = "geo-cloudfront"))] +#[path = "support/celld.rs"] +mod celld_fixture; +#[cfg(all(feature = "celld-ec", feature = "geo-cloudfront"))] +use base64::{Engine as _, engine::general_purpose::STANDARD}; +#[cfg(all(feature = "celld-ec", feature = "geo-cloudfront"))] +use trusted_server_core::ec::kv_backend::{ + EcKvStore as _, EcKvWrite, EcKvWriteMode, EcKvWriteOutcome, +}; + const ADMIN_PASSWORD: &str = "native-admin-password-32-bytes-example"; const PROXY_SECRET: &str = "native-proxy-secret-32-bytes-example"; const EC_PASSPHRASE: &str = "native-ec-passphrase-32-bytes-example"; +#[cfg(all(feature = "celld-ec", feature = "geo-cloudfront"))] +fn ec_response_cookie(response: &Response) -> Option { + response + .headers() + .get_all("set-cookie") + .iter() + .filter_map(|value| value.to_str().ok()) + .find_map(|value| { + value.strip_prefix("ts-ec=").map(|value| { + value + .split(';') + .next() + .expect("should contain cookie value") + .to_owned() + }) + }) +} + +#[cfg(all(feature = "celld-ec", feature = "geo-cloudfront"))] +#[test] +#[ignore = "requires CELLD_EC_TEST_BINARY and celld-ec,geo-cloudfront features; real disposable servers only"] +fn real_celld_envelope_drives_native_http_issuance_withdrawal_restart_and_outage() { + let mut celld = celld_fixture::CelldFixture::new(); + let origin = origin::Origin::start("celld-ec-http"); + let mut prepared = Prepared::new(&origin.base_url); + let mut storage = + serde_json::to_value(celld.settings()).expect("should serialize pure selected binding"); + storage["provider"] = json!("celld"); + prepared.data["ec"]["storage"] = storage; + prepared.data["ec"]["partners"] = json!([{"name":"Fixture partner","source_domain":"partner.example.com","bidstream_enabled":true,"pull_sync_enabled":false}]); + prepared.publish(); + let profile = prepared.workspace.path().join("native-geo.toml"); + fs::write(&profile,"version = 1\n[geo]\nprovider = \"cloudfront\"\nalb_peer_cidrs = [\"127.0.0.1/32\"]\norigin_credential_header_name = \"x-origin-synthetic-fixture\"\n").expect("should write synthetic local protected-ingress profile"); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("should drive typed record inspection"); + let store = runtime.block_on(celld.store()); + let client = client(None); + let document = |server: &Running, country: &str, cookie: Option<&str>, withdrawal: bool| { + let mut request = client + .get(format!("{}/article", server.base_url())) + .header("host", "publisher.example.com") + .header( + "forwarded", + "for=198.51.100.24;proto=https;host=publisher.example.com", + ) + .header("accept", "text/html") + .header("sec-fetch-dest", "document") + .header("sec-fetch-mode", "navigate") + .header( + "user-agent", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/146.0.0.0 Safari/537.36", + ) + .header("cloudfront-viewer-country", country) + .header("cloudfront-viewer-country-region", "TN") + .header("x-origin-synthetic-fixture", "synthetic-fixture-only"); + if let Some(cookie) = cookie { + request = request.header("cookie", cookie); + } + if withdrawal { + request = request.header("sec-gpc", "1"); + } + request + .send() + .expect("should complete actual native publisher HTTP request") + }; + let mut app = prepared.spawn_with_geo(true, Some(&profile)); + app.ready(); + let issued = document(&app, "AU", None, false); + assert_eq!(issued.status().as_u16(), 200); + assert_eq!( + issued + .headers() + .get("x-geo-country") + .expect("should admit checked synthetic geography"), + "AU" + ); + let ec = ec_response_cookie(&issued) + .expect("should issue only with confirmed real record persistence"); + let created = runtime + .block_on(store.lookup(&ec)) + .expect("should read actual persisted identity") + .expect("should retain issued identity"); + let eids = STANDARD.encode( + json!([{"source":"partner.example.com","uids":[{"id":"fixture-uid","atype":1}]}]) + .to_string(), + ); + let cookie = format!("ts-ec={ec}; ts-eids={eids}"); + let updated = document(&app, "AU", Some(&cookie), false); + assert_eq!(updated.status().as_u16(), 200); + let updated = runtime + .block_on(store.lookup(&ec)) + .expect("should read real native EID update") + .expect("should retain updated identity"); + let row: Value = + serde_json::from_slice(&updated.body).expect("should decode persisted graph schema"); + assert_eq!(row["ids"]["partner.example.com"]["uid"], "fixture-uid"); + assert!(updated.generation > created.generation); + drop(app); + app = prepared.spawn_with_geo(true, Some(&profile)); + app.ready(); + assert!( + ec_response_cookie(&document(&app, "AU", Some(&cookie), false)).is_none(), + "should reuse identity across application recreation" + ); + let withdrawn = document(&app, "US", Some(&cookie), true); + assert_eq!(ec_response_cookie(&withdrawn).as_deref(), Some("")); + let tombstone = runtime + .block_on(store.lookup(&ec)) + .expect("should read acknowledged native withdrawal") + .expect("should retain tombstone"); + let row: Value = + serde_json::from_slice(&tombstone.body).expect("should decode actual tombstone"); + assert_eq!(row["consent"]["ok"], false); + celld.stop(); + assert!( + ec_response_cookie(&document(&app, "AU", None, false)).is_none(), + "should not issue unsupported cookie during real storage outage" + ); + drop(app); + celld.restart(); + prepared.data["ec"]["storage"]["endpoint"] = json!(celld.endpoint); + prepared.publish(); + app = prepared.spawn_with_geo(true, Some(&profile)); + app.ready(); + assert!( + ec_response_cookie(&document(&app, "AU", Some(&cookie), false)).is_none(), + "should not revive withdrawal after server and application restart" + ); + let store = runtime.block_on(celld.store()); + let restored = runtime + .block_on(store.lookup(&ec)) + .expect("should read recovered tombstone") + .expect("should restore withdrawal"); + assert_eq!(restored.body, tombstone.body); + assert_eq!( + runtime + .block_on(store.insert( + &ec, + EcKvWrite { + body: std::str::from_utf8(&updated.body).expect("should decode old entry"), + metadata: "{}", + ttl: Duration::from_secs(60), + mode: EcKvWriteMode::IfGenerationMatch(updated.generation) + } + )) + .expect("should reject stale writer"), + EcKvWriteOutcome::PreconditionFailed + ); +} + struct Prepared { workspace: TempDir, data: Value, @@ -102,6 +264,14 @@ impl Prepared { } fn spawn(&self, trusted_proxy: bool) -> Running { + self.spawn_with_geo(trusted_proxy, None) + } + + fn spawn_with_geo( + &self, + trusted_proxy: bool, + geo_profile: Option<&std::path::Path>, + ) -> Running { let port = TcpListener::bind("127.0.0.1:0") .expect("should reserve isolated port") .local_addr() @@ -163,6 +333,9 @@ impl Prepared { .env("EDGEZERO__ADAPTER__TRUSTED_PROXY_CIDRS", "127.0.0.1/32") .env("EDGEZERO__ADAPTER__FORWARDING_HEADER_FAMILY", "forwarded"); } + if let Some(profile) = geo_profile { + command.env("TS_NATIVE_GEO_PROFILE_FILE", profile); + } Running { child: command .spawn() diff --git a/crates/trusted-server-adapter-axum/tests/native_ec.rs b/crates/trusted-server-adapter-axum/tests/native_ec.rs index 4774d641ad..fff640dad9 100644 --- a/crates/trusted-server-adapter-axum/tests/native_ec.rs +++ b/crates/trusted-server-adapter-axum/tests/native_ec.rs @@ -27,6 +27,125 @@ use trusted_server_core::platform::{ use trusted_server_core::settings::Settings; use trusted_server_core::test_support::nextjs_auction::{self, NextJsAuctionOrigin}; +#[cfg(feature = "celld-ec")] +#[path = "support/celld.rs"] +mod celld_fixture; + +#[cfg(feature = "celld-ec")] +#[tokio::test(flavor = "current_thread")] +#[ignore = "requires CELLD_EC_TEST_BINARY pointing to the built real conditional-record server"] +async fn real_celld_persists_routes_withdrawal_restart_and_outage_without_fallback() { + let mut celld = celld_fixture::CelldFixture::new(); + let store = celld.store().await; + let app = allowed_router(store.clone()); + let issued = call(&app, request(Method::GET, "/article", None)).await; + let ec = + ec_cookie(&issued).expect("should issue cookie only after real record acknowledgement"); + let original = store + .lookup(&ec) + .await + .expect("should read real record") + .expect("should retain issued identity"); + assert!( + store + .key_exists(&ec) + .await + .expect("should strongly confirm exact identity") + ); + let eids = BASE64.encode( + json!([{"source":"partner.example.com","uids":[{"id":"celld-fixture-uid","atype":1}]}]) + .to_string(), + ); + let cookie = format!("ts-ec={ec}; ts-eids={eids}"); + let returned = call(&app, request(Method::GET, "/article", Some(&cookie))).await; + assert_eq!(returned.status(), 200); + let updated = store + .lookup(&ec) + .await + .expect("should read updated record") + .expect("should retain updated identity"); + assert!(updated.generation > original.generation); + let data: serde_json::Value = + serde_json::from_slice(&updated.body).expect("should decode unchanged graph schema"); + assert_eq!( + data["ids"]["partner.example.com"]["uid"], + "celld-fixture-uid" + ); + celld.restart(); + let store = celld.store().await; + let restored = store + .lookup(&ec) + .await + .expect("should read after actual celld restart") + .expect("should restore identity"); + assert_eq!(restored.body, updated.body); + assert_eq!(restored.metadata, updated.metadata); + assert_eq!(restored.generation, updated.generation); + let regulated = router( + store.clone(), + Some(GeoInfo::country_region("US", Some("TN".to_owned()))), + ); + let mut withdrawal = request(Method::GET, "/article", Some(&cookie)); + withdrawal.headers_mut().insert( + "sec-gpc", + "1".parse().expect("should set synthetic withdrawal signal"), + ); + let withdrawn = call(®ulated, withdrawal).await; + assert_eq!(ec_cookie(&withdrawn).as_deref(), Some("")); + let tombstone = store + .lookup(&ec) + .await + .expect("should read actual tombstone") + .expect("should retain actual tombstone"); + let data: serde_json::Value = + serde_json::from_slice(&tombstone.body).expect("should decode tombstone"); + assert_eq!(data["consent"]["ok"], false); + assert_eq!( + store + .insert( + &ec, + EcKvWrite { + body: std::str::from_utf8(&updated.body).expect("should decode old entry"), + metadata: "{}", + ttl: Duration::from_secs(60), + mode: EcKvWriteMode::IfGenerationMatch(updated.generation) + } + ) + .await + .expect("should classify stale CAS"), + EcKvWriteOutcome::PreconditionFailed + ); + celld.restart(); + let store = celld.store().await; + assert_eq!( + store + .lookup(&ec) + .await + .expect("should read recovered withdrawal") + .expect("should retain withdrawal") + .body, + tombstone.body + ); + let app = allowed_router(store.clone()); + assert!( + ec_cookie(&call(&app, request(Method::GET, "/article", Some(&cookie))).await).is_none(), + "should not resurrect withdrawal" + ); + celld.stop(); + let failed = call(&app, request(Method::GET, "/article", None)).await; + assert!( + ec_cookie(&failed).is_none(), + "should not mint unsupported cookie on real storage outage" + ); + assert!( + store.key_exists(&ec).await.is_err(), + "should not turn outage into false absence" + ); + trusted_server_adapter_axum::celld_ec::CelldEcKv::prepare(&celld.settings(), None) + .await + .expect_err("should fail selected-provider preparation without fallback"); +} + #[derive(Clone, Default)] struct Store { rows: Arc>>, @@ -157,7 +276,10 @@ fn settings() -> Settings { settings } -fn router(store: Store, geo: Option) -> RouterService { +fn router( + store: S, + geo: Option, +) -> RouterService { let services = RuntimeServices::builder() .config_store(Arc::new(UnusedStores)) .secret_store(Arc::new(UnusedStores)) @@ -173,7 +295,7 @@ fn router(store: Store, geo: Option) -> RouterService { TrustedServerApp::routes_with_settings_services_and_ec_store(settings(), services, store) .expect("should construct native EC fixture router") } -fn allowed_router(store: Store) -> RouterService { +fn allowed_router(store: S) -> RouterService { router(store, Some(GeoInfo::country_region("AU", None))) } fn request(method: Method, path: &str, cookie: Option<&str>) -> Request { diff --git a/crates/trusted-server-adapter-axum/tests/startup.rs b/crates/trusted-server-adapter-axum/tests/startup.rs index 09f4b2144a..f5666bbda5 100644 --- a/crates/trusted-server-adapter-axum/tests/startup.rs +++ b/crates/trusted-server-adapter-axum/tests/startup.rs @@ -10,6 +10,8 @@ use std::thread; use std::time::{Duration, Instant}; use edgezero_core::app::{Hooks as _, StoreMetadata}; +use edgezero_core::blob_envelope::BlobEnvelope; +use serde_json::json; use trusted_server_adapter_axum::app::TrustedServerApp; use trusted_server_core::config_payload::{DEFAULT_CONFIG_STORE_ID, DEFAULT_SECRET_STORE_ID}; @@ -18,6 +20,41 @@ use trusted_server_core::config_payload::{DEFAULT_CONFIG_STORE_ID, DEFAULT_SECRE const ENVELOPE: &str = r#"{"data":{"ec":{"passphrase":"ec_key"},"handlers":[{"password":"admin_key","path":"^/_ts/admin","username":"admin"}],"publisher":{"cookie_domain":".publisher.example.com","domain":"publisher.example.com","origin_url":"https://origin.publisher.example.com","proxy_secret":"proxy_key"},"request_signing":null},"sha256":"f4dc03017deb24bfd589efde7dce0f9f8a20d952c28ebcbdb6141b08661cd137","version":1,"generated_at":"2026-10-20T00:00:00Z"}"#; static NEXT_DIRECTORY: AtomicU64 = AtomicU64::new(0); +#[cfg(feature = "celld-ec")] +#[path = "support/celld.rs"] +mod celld_fixture; + +#[cfg(feature = "celld-ec")] +#[tokio::test(flavor = "current_thread")] +#[ignore = "requires CELLD_EC_TEST_BINARY pointing to the built real conditional-record server"] +async fn real_celld_provider_prepares_from_published_envelope_and_fails_closed_on_outage() { + let mut celld = celld_fixture::CelldFixture::new(); + for _ in 0..2 { + let storage = + serde_json::to_value(celld.settings()).expect("should serialize pure celld settings"); + let mut storage = storage; + storage["provider"] = json!("celld"); + let mut server = ServerFixture::new(Some(&storage_envelope(storage))); + server.spawn(false, false); + server.wait_ready(); + let store = celld.store().await; + assert!( + !trusted_server_core::ec::kv_backend::EcKvStore::key_exists(&store, "never-issued") + .await + .expect("should read strongly through prepared typed backend") + ); + drop(server); + celld.restart(); + } + let mut storage = + serde_json::to_value(celld.settings()).expect("should serialize final binding"); + storage["provider"] = json!("celld"); + celld.stop(); + let mut server = ServerFixture::new(Some(&storage_envelope(storage))); + server.spawn(false, false); + server.assert_failed_before_listen("native EC celld preparation failed"); +} + struct ServerFixture { root: PathBuf, addr: SocketAddr, @@ -60,6 +97,16 @@ impl ServerFixture { missing_secret: bool, short_passphrase: bool, profile: Option<&str>, + ) { + self.spawn_inputs(missing_secret, short_passphrase, profile, None); + } + + fn spawn_inputs( + &mut self, + missing_secret: bool, + short_passphrase: bool, + profile: Option<&str>, + legacy: Option<&str>, ) { let mut command = Command::new(env!("CARGO_BIN_EXE_trusted-server-axum")); command @@ -102,6 +149,9 @@ impl ServerFixture { }, ); } + if let Some(value) = legacy { + command.env("TS_EC_DYNAMODB_ENABLED", value); + } self.child = Some( command .spawn() @@ -230,6 +280,52 @@ impl Drop for ServerFixture { } } +fn storage_envelope(storage: serde_json::Value) -> String { + let mut envelope: serde_json::Value = + serde_json::from_str(ENVELOPE).expect("should decode fixture envelope"); + let mut data = envelope["data"].take(); + data["ec"]["storage"] = storage; + serde_json::to_string(&BlobEnvelope::new(data, "2026-01-01T00:00:00Z".to_owned())) + .expect("should encode existing envelope format") +} + +#[test] +fn active_legacy_ec_environment_cannot_disable_or_override_published_storage() { + let configured = storage_envelope( + json!({"provider":"celld","endpoint":"http://127.0.0.1:7780","store":"identity","access":{"mode":"trusted-network"}}), + ); + for envelope in [ENVELOPE, configured.as_str()] { + let mut server = ServerFixture::new(Some(envelope)); + server.spawn_inputs(false, false, None, Some("true")); + server.assert_failed_before_listen( + "publish ec.storage and remove legacy EC environment selection", + ); + } +} + +#[test] +fn explicit_unsupported_and_uncompiled_ec_providers_fail_before_listen() { + let cases=[ + Some((json!({"provider":"fastly","store":"exact-binding"}),"unsupported by the selected serving runtime")), + (!cfg!(feature="aws-dynamodb-ec")).then(||(json!({"provider":"dynamodb","region":"us-east-1","table":"example-identity","namespace":"fixture"}),"aws-dynamodb-ec build feature is unavailable")), + (!cfg!(feature="celld-ec")).then(||(json!({"provider":"celld","endpoint":"http://127.0.0.1:7780","store":"identity","access":{"mode":"trusted-network"}}),"celld-ec build feature is unavailable")), + ]; + for (storage, expected) in cases.into_iter().flatten() { + let mut server = ServerFixture::new(Some(&storage_envelope(storage))); + server.spawn(false, false); + server.assert_failed_before_listen(expected); + } +} + +#[cfg(feature = "celld-ec")] +#[test] +fn selected_celld_captures_only_its_explicit_bootstrap_reference_before_network() { + let storage = json!({"provider":"celld","endpoint":"https://127.0.0.1:7780","store":"identity","access":{"mode":"authenticated","token_env":"FIXTURE_MISSING_CELLD_TOKEN"}}); + let mut server = ServerFixture::new(Some(&storage_envelope(storage))); + server.spawn(false, false); + server.assert_failed_before_listen("selected EC celld bootstrap token is unavailable"); +} + #[test] fn missing_config_fails_before_listen_without_legacy_fallback() { let mut server = ServerFixture::new(None); diff --git a/crates/trusted-server-adapter-axum/tests/support/celld.rs b/crates/trusted-server-adapter-axum/tests/support/celld.rs new file mode 100644 index 0000000000..9deab739d7 --- /dev/null +++ b/crates/trusted-server-adapter-axum/tests/support/celld.rs @@ -0,0 +1,105 @@ +//! Disposable real celld, selected explicitly by an external-test prerequisite. + +use std::io::{BufRead as _, BufReader}; +use std::path::PathBuf; +use std::process::{Child, Command, Stdio}; +use std::time::Duration; + +use edgezero_store_celld::settings::{CelldAccess, CelldKvSettings}; +use serde_json::json; +use trusted_server_adapter_axum::celld_ec::CelldEcKv; + +pub struct CelldFixture { + root: tempfile::TempDir, + binary: PathBuf, + child: Option, + pub endpoint: String, +} + +impl CelldFixture { + pub fn new() -> Self { + let binary = PathBuf::from(std::env::var_os("CELLD_EC_TEST_BINARY").expect( + "should set CELLD_EC_TEST_BINARY to the built conditional-record celld executable", + )); + assert!( + binary.is_absolute() && binary.is_file(), + "should supply an existing absolute celld executable path" + ); + let root = tempfile::tempdir().expect("should create disposable celld directory"); + let policy = json!({"version":1,"listen":"127.0.0.1:0", + "stores":[{"alias":"identity","namespace":"fixture-ec"}], + "access":{"mode":"trusted-network","grants":[{"store":"identity","operations":["read","write","delete","list"]}]}}); + std::fs::write( + root.path().join("policy.json"), + serde_json::to_vec(&policy).expect("should encode fixture policy"), + ) + .expect("should write disposable fixture policy"); + let mut fixture = Self { + root, + binary, + child: None, + endpoint: String::new(), + }; + fixture.start(); + fixture + } + fn start(&mut self) { + let mut child = Command::new(&self.binary) + .env_clear() + .env("CELLD_TEST_DATA_DIR", self.root.path().join("node-cache")) + .args([ + "--listen", + "127.0.0.1:0", + "--internal-listen", + "127.0.0.1:0", + "--store-api-config", + ]) + .arg(self.root.path().join("policy.json")) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() + .expect("should start disposable real celld"); + let stdout = child.stdout.take().expect("should capture celld readiness"); + let (send, receive) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(stdout).lines().map_while(Result::ok) { + if let Some((_, address)) = line.split_once("celld scoped stores listening on ") { + let _ = send.send(format!("http://{}", address.trim())); + } + } + }); + self.child = Some(child); + self.endpoint = receive + .recv_timeout(Duration::from_secs(30)) + .expect("should observe real celld readiness"); + } + pub fn stop(&mut self) { + if let Some(mut child) = self.child.take() { + let _ = child.kill(); + let _ = child.wait(); + } + } + pub fn restart(&mut self) { + self.stop(); + self.start(); + } + pub fn settings(&self) -> CelldKvSettings { + CelldKvSettings { + endpoint: self.endpoint.clone(), + store: "identity".to_owned(), + access: CelldAccess::TrustedNetwork { + allow_remote_http: false, + }, + } + } + pub async fn store(&self) -> CelldEcKv { + CelldEcKv::prepare(&self.settings(), None) + .await + .expect("should prepare typed records against real celld") + } +} +impl Drop for CelldFixture { + fn drop(&mut self) { + self.stop(); + } +} diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 3daf3202b3..185defea55 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -203,6 +203,9 @@ fn build_state_with_services( settings: Settings, services: Option, ) -> Result, Report> { + settings + .ec + .resolve_storage(trusted_server_core::ec::storage::EcRuntime::Cloudflare)?; let plan = Arc::new(compile_auction_plan(&settings)?); plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Cloudflare)?; let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; @@ -812,6 +815,32 @@ fn build_router(state: &Arc) -> RouterService { mod tests { use super::*; + #[test] + fn explicit_ec_storage_is_rejected_before_route_or_provider_work() { + for storage in [ + serde_json::json!({"provider":"fastly","store":"exact-fixture-binding"}), + serde_json::json!({"provider":"dynamodb","region":"us-east-1","table":"example-identity","namespace":"fixture"}), + serde_json::json!({"provider":"celld","endpoint":"http://127.0.0.1:7780","store":"identity","access":{"mode":"trusted-network"}}), + ] { + let mut settings = Settings::default(); + settings.ec.storage = + Some(serde_json::from_value(storage).expect("should parse pure provider settings")); + let error = build_state_with_settings(settings) + .err() + .expect("should reject unsupported EC provider before route work"); + assert!(format!("{error:?}").contains("ec.storage.provider is unsupported")); + } + let mut settings = Settings::default(); + settings.ec.ec_store = Some("legacy-inactive-binding".to_owned()); + assert!( + settings + .ec + .resolve_storage(trusted_server_core::ec::storage::EcRuntime::Cloudflare) + .expect("should accept inactive legacy field") + .is_none() + ); + } + fn aps_profile_settings() -> Settings { let mut settings = Settings::from_toml( r#" diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index e0b4d571ec..0f021be75c 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -221,6 +221,9 @@ pub(crate) fn build_state_from_settings( ) -> Result, Report> { warn_if_certificate_check_disabled(&settings); + settings + .ec + .resolve_storage(trusted_server_core::ec::storage::EcRuntime::Fastly)?; let plan = Arc::new(compile_auction_plan(&settings)?); plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Fastly)?; let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; @@ -451,7 +454,11 @@ fn build_ec_request_state( // Bot gate: suppress KV-backed EC writes for unrecognized clients, except // consent withdrawals. Revocations keep the write path so tombstones stay // authoritative even for privacy-extension-heavy clients. - let kv_graph = crate::maybe_identity_graph(settings); + let (kv_graph, storage_error) = match crate::maybe_identity_graph(settings) { + Ok(graph) => (graph, None), + Err(error) => (None, Some(error)), + }; + let setup_error = setup_error.or(storage_error); let finalize_kv_graph = if setup_error.is_none() && (is_real_browser || ec_consent_withdrawn(ec_context.consent())) { @@ -627,7 +634,7 @@ async fn execute_named( // Deliberately do not use an EC request-state graph: that // copy is bot-gated, while operators use curl for this // authenticated diagnostic. - let kv = crate::maybe_identity_graph(&state.settings); + let kv = crate::maybe_identity_graph(&state.settings)?; handle_admin_ec_lookup(kv.as_ref(), ®istry, &req).await } NamedRouteHandler::AdminEidsLookup => handle_admin_eids_lookup(®istry, &req), diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 0b54fb59ca..e2cb62ae9a 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -52,6 +52,7 @@ use crate::middleware::{HEADER_X_TS_FINALIZED, apply_finalize_headers, resolve_g use crate::platform::{FastlyPlatformGeo, client_info_from_request}; use crate::rate_limiter::{FastlyRateLimiter, RATE_COUNTER_NAME}; use crate::sandbox::{RetainedApp, Sandbox, SandboxCounters, ServeMode, StartupDiagnostics}; +use trusted_server_core::ec::storage::{EcRuntime, EcStorageSelection}; // Only the reuse path builds a serving loop, so the retirement snapshots have // no consumer in the default build. #[cfg(feature = "reusable-sandbox")] @@ -615,7 +616,7 @@ async fn apply_edgezero_ec_finalize( ) -> Result> { let partner_registry = PartnerRegistry::from_config(&settings.ec.partners)?; let finalize_kv_graph = if ec_state.use_finalize_kv { - maybe_identity_graph(settings) + maybe_identity_graph(settings)? } else { None }; @@ -749,12 +750,16 @@ fn build_ja4_debug_response(req: &FastlyRequest) -> FastlyResponse { .with_body(body) } -pub(crate) fn maybe_identity_graph(settings: &Settings) -> Option { - settings - .ec - .ec_store - .as_ref() - .map(|store_name| KvIdentityGraph::new(FastlyEcKvStore::new(store_name))) +pub(crate) fn maybe_identity_graph( + settings: &Settings, +) -> Result, Report> { + match settings.ec.resolve_storage(EcRuntime::Fastly)? { + Some(EcStorageSelection::Fastly(store_name)) => { + Ok(Some(KvIdentityGraph::new(FastlyEcKvStore::new(store_name)))) + } + None => Ok(None), + _ => unreachable!("Fastly resolution should select only Fastly storage"), + } } /// Constructs a `KvIdentityGraph` from settings, or returns an error if the @@ -762,13 +767,12 @@ pub(crate) fn maybe_identity_graph(settings: &Settings) -> Option Result> { - let store_name = settings.ec.ec_store.as_deref().ok_or_else(|| { + maybe_identity_graph(settings)?.ok_or_else(|| { Report::new(TrustedServerError::KvStore { - store_name: "ec.ec_store".to_owned(), - message: "ec.ec_store is not configured".to_owned(), + store_name: "ec.storage".to_owned(), + message: "EC identity storage is not configured".to_owned(), }) - })?; - Ok(KvIdentityGraph::new(FastlyEcKvStore::new(store_name))) + }) } /// Extracts a named cookie value from the request's `Cookie` header. @@ -1034,6 +1038,36 @@ mod tests { assert!(response.headers().get("cdn-cache-control").is_none()); } + #[test] + fn fastly_graph_selection_preserves_legacy_and_new_binding_input() { + for storage in [ + None, + Some(serde_json::json!({"store":"exact-fixture-binding"})), + Some(serde_json::json!({"provider":"fastly","store":"exact-fixture-binding"})), + ] { + let mut settings = diagnostics_settings(); + settings.ec.storage = storage + .map(|value| serde_json::from_value(value).expect("should parse provider config")); + if settings.ec.storage.is_none() { + settings.ec.ec_store = Some("exact-fixture-binding".to_owned()); + } + assert!(matches!( + settings + .ec + .resolve_storage(EcRuntime::Fastly) + .expect("should resolve Fastly input"), + Some(EcStorageSelection::Fastly("exact-fixture-binding")) + )); + assert!( + maybe_identity_graph(&settings) + .expect("should construct graph lazily") + .is_some() + ); + require_identity_graph(&settings) + .expect("should require the configured binding without opening it early"); + } + } + fn diagnostics_settings() -> Settings { Settings::from_toml( r#" diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 18b1c4468c..25864ba62a 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -70,6 +70,9 @@ fn build_state_with_services( settings: Settings, services: Option, ) -> Result, Report> { + settings + .ec + .resolve_storage(trusted_server_core::ec::storage::EcRuntime::Spin)?; let plan = Arc::new(compile_auction_plan(&settings)?); plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Spin)?; let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; @@ -876,6 +879,32 @@ fn build_router(state: Option>) -> RouterService { mod tests { use super::*; + #[test] + fn explicit_ec_storage_is_rejected_before_route_or_provider_work() { + for storage in [ + serde_json::json!({"provider":"fastly","store":"exact-fixture-binding"}), + serde_json::json!({"provider":"dynamodb","region":"us-east-1","table":"example-identity","namespace":"fixture"}), + serde_json::json!({"provider":"celld","endpoint":"http://127.0.0.1:7780","store":"identity","access":{"mode":"trusted-network"}}), + ] { + let mut settings = Settings::default(); + settings.ec.storage = + Some(serde_json::from_value(storage).expect("should parse pure provider settings")); + let error = build_state_with_settings(settings) + .err() + .expect("should reject unsupported EC provider before route work"); + assert!(format!("{error:?}").contains("ec.storage.provider is unsupported")); + } + let mut settings = Settings::default(); + settings.ec.ec_store = Some("legacy-inactive-binding".to_owned()); + assert!( + settings + .ec + .resolve_storage(trusted_server_core::ec::storage::EcRuntime::Spin) + .expect("should accept inactive legacy field") + .is_none() + ); + } + fn multi_provider_settings() -> Settings { let mut settings = Settings::from_toml( r#" diff --git a/crates/trusted-server-core/Cargo.toml b/crates/trusted-server-core/Cargo.toml index 0a8d052805..bed1dbac54 100644 --- a/crates/trusted-server-core/Cargo.toml +++ b/crates/trusted-server-core/Cargo.toml @@ -23,6 +23,7 @@ cssparser = { workspace = true } derive_more = { workspace = true } ed25519-dalek = { workspace = true } edgezero-core = { workspace = true } +edgezero-store-celld = { workspace = true } error-stack = { workspace = true } flate2 = { workspace = true } futures = { workspace = true } diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index a59cc71eb0..db87be786a 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -50,6 +50,7 @@ pub mod pull_sync; pub(crate) mod pull_sync_marker; pub mod rate_limiter; pub mod registry; +pub mod storage; /// Characters of an identifier kept when redacting it for a log. const LOG_ID_PREFIX_CHARS: usize = 8; diff --git a/crates/trusted-server-core/src/ec/storage.rs b/crates/trusted-server-core/src/ec/storage.rs new file mode 100644 index 0000000000..4c7555ce39 --- /dev/null +++ b/crates/trusted-server-core/src/ec/storage.rs @@ -0,0 +1,223 @@ +//! Pure EC persistence configuration and serving-runtime resolution. + +use edgezero_store_celld::settings::CelldKvSettings; +use error_stack::Report; +use serde::{Deserialize, Deserializer, Serialize}; + +use crate::error::TrustedServerError; +use crate::settings::Ec; + +/// The serving adapter, independent of the host running configuration tooling. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EcRuntime { + /// Fastly Compute. + Fastly, + /// Native Axum. + Native, + /// Cloudflare Workers. + Cloudflare, + /// Fermyon Spin. + Spin, +} + +/// A named Fastly binding without an inferred provider in published config. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct FastlyEcStorage { + /// Exact Fastly binding name. + pub store: String, +} + +/// Single-region `DynamoDB` configuration. Credentials remain outside app config. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct DynamoDbEcStorage { + /// Explicit AWS region. + pub region: String, + /// Existing retained table. + pub table: String, + /// Exact stable namespace, without normalization. + pub namespace: String, + /// Explicit loopback emulator endpoint, never a production override. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub local_endpoint: Option, +} + +/// Explicit provider configuration, validated without SDKs or network access. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "provider", rename_all = "lowercase", deny_unknown_fields)] +pub enum EcStorageProvider { + /// Fastly's conditional KV binding. + Fastly(FastlyEcStorage), + /// Native single-region `DynamoDB`. + DynamoDb(DynamoDbEcStorage), + /// Native scoped conditional records, not ordinary KV. + Celld(CelldKvSettings), +} + +/// Published storage input. An omitted provider stays omitted on serialization. +#[derive(Debug, Clone, Serialize)] +#[serde(untagged)] +pub enum EcStorage { + /// Explicit provider selection. + Explicit(EcStorageProvider), + /// Fastly shorthand, resolved only when the serving runtime is known. + Implicit(FastlyEcStorage), +} + +impl<'de> Deserialize<'de> for EcStorage { + fn deserialize>(deserializer: D) -> Result { + let value = serde_json::Value::deserialize(deserializer)?; + if value.get("provider").is_some() { + // Report field names, never raw serde input that could contain an + // endpoint or an accidentally supplied credential value. + let fields = match value.get("provider").and_then(serde_json::Value::as_str) { + Some("fastly") => { + "ec.storage.store is required for fastly; other provider fields are forbidden" + } + Some("dynamodb") => { + "ec.storage.region, ec.storage.table and ec.storage.namespace are required for dynamodb; only ec.storage.local_endpoint is optional" + } + Some("celld") => { + "ec.storage.endpoint, ec.storage.store and ec.storage.access are required for celld; access requires a valid mode and credential reference, not a token value" + } + _ => "ec.storage.provider must be fastly, dynamodb or celld", + }; + serde_json::from_value(value) + .map(Self::Explicit) + .map_err(|_| serde::de::Error::custom(fields)) + } else { + serde_json::from_value(value).map(Self::Implicit).map_err(|_| { + serde::de::Error::custom("ec.storage.store is required; provider-specific fields require an explicit provider") + }) + } + } +} + +/// Validated selection for adapter-owned provider construction. +#[derive(Debug, Clone, Copy)] +pub enum EcStorageSelection<'a> { + /// Exact Fastly binding. + Fastly(&'a str), + /// Native `DynamoDB` settings. + DynamoDb(&'a DynamoDbEcStorage), + /// Native celld settings, without a captured token. + Celld(&'a CelldKvSettings), +} + +fn invalid(message: &'static str) -> Report { + Report::new(TrustedServerError::Configuration { + message: message.to_owned(), + }) +} + +impl DynamoDbEcStorage { + /// Validate exact bindings and the explicit local-only endpoint shape. + /// + /// # Errors + /// Returns field-oriented configuration errors without echoing input values. + pub fn validate(&self) -> Result<(), Report> { + if self.region.is_empty() || self.region.chars().any(char::is_whitespace) { + return Err(invalid( + "ec.storage.region is required and must not contain whitespace", + )); + } + if !(3..=255).contains(&self.table.len()) + || !self + .table + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"_.-".contains(&b)) + { + return Err(invalid( + "ec.storage.table must be a valid DynamoDB table name", + )); + } + if self.namespace.is_empty() + || self.namespace.len() > 512 + || self.namespace.chars().any(char::is_control) + { + return Err(invalid( + "ec.storage.namespace must contain 1 through 512 bytes without control characters", + )); + } + if let Some(endpoint) = &self.local_endpoint { + let url=url::Url::parse(endpoint).map_err(|_|invalid("ec.storage.local_endpoint must be a loopback HTTP root URL with an explicit port"))?; + if url.scheme() != "http" + || url.host_str() != Some("127.0.0.1") + || url.port().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.path() != "/" + || url.query().is_some() + || url.fragment().is_some() + { + return Err(invalid( + "ec.storage.local_endpoint must be a loopback HTTP root URL with an explicit port", + )); + } + } + Ok(()) + } +} + +impl Ec { + /// Validate publication shape without selecting according to the CLI host. + /// + /// # Errors + /// Rejects simultaneous legacy/new input or malformed provider bindings. + pub fn validate_storage(&self) -> Result<(), Report> { + if self.ec_store.is_some() && self.storage.is_some() { + return Err(invalid( + "ec.ec_store and ec.storage cannot both be configured", + )); + } + match &self.storage { + Some( + EcStorage::Implicit(config) + | EcStorage::Explicit(EcStorageProvider::Fastly(config)), + ) if config.store.is_empty() => Err(invalid("ec.storage.store must not be empty")), + Some(EcStorage::Explicit(EcStorageProvider::DynamoDb(config))) => config.validate(), + Some(EcStorage::Explicit(EcStorageProvider::Celld(config))) => config + .validate() + .map_err(|_| invalid("ec.storage celld endpoint, store or access is invalid")), + _ => Ok(()), + } + } + + /// Resolve validated provider input against the actual serving adapter. + /// + /// # Errors + /// Rejects unsupported explicit providers and omitted providers off Fastly. + pub fn resolve_storage( + &self, + runtime: EcRuntime, + ) -> Result>, Report> { + self.validate_storage()?; + let Some(storage) = &self.storage else { + return Ok(if runtime == EcRuntime::Fastly { + self.ec_store.as_deref().map(EcStorageSelection::Fastly) + } else { + None + }); + }; + match (storage, runtime) { + ( + EcStorage::Implicit(config) + | EcStorage::Explicit(EcStorageProvider::Fastly(config)), + EcRuntime::Fastly, + ) => Ok(Some(EcStorageSelection::Fastly(&config.store))), + (EcStorage::Explicit(EcStorageProvider::DynamoDb(config)), EcRuntime::Native) => { + Ok(Some(EcStorageSelection::DynamoDb(config))) + } + (EcStorage::Explicit(EcStorageProvider::Celld(config)), EcRuntime::Native) => { + Ok(Some(EcStorageSelection::Celld(config))) + } + (EcStorage::Implicit(_), _) => { + Err(invalid("ec.storage.provider is required outside Fastly")) + } + _ => Err(invalid( + "ec.storage.provider is unsupported by the selected serving runtime", + )), + } + } +} diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 06e7ed1fda..f6e3f90159 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -548,6 +548,10 @@ pub struct Ec { #[serde(default)] pub ec_store: Option, + /// Explicit persistence provider or the Fastly store-only shorthand. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub storage: Option, + /// Maximum number of concurrent pull-sync requests. #[serde(default = "Ec::default_pull_sync_concurrency")] pub pull_sync_concurrency: usize, @@ -3127,6 +3131,7 @@ impl Settings { /// opportunity slot is invalid, or if /// [`AuctionDebugCommentOptions::metadata_keys`] names an unsupported key. pub fn prepare_runtime(&mut self) -> Result<(), Report> { + self.ec.validate_storage()?; self.image_optimizer.prepare_runtime()?; self.cache.prepare_runtime()?; self.proxy.prepare_runtime()?; diff --git a/crates/trusted-server-core/tests/ec_storage_config.rs b/crates/trusted-server-core/tests/ec_storage_config.rs new file mode 100644 index 0000000000..24e7a8ebd4 --- /dev/null +++ b/crates/trusted-server-core/tests/ec_storage_config.rs @@ -0,0 +1,190 @@ +//! Provider input remains pure publication data, not runtime or credential lookup. + +use edgezero_core::blob_envelope::BlobEnvelope; +use serde_json::json; +use trusted_server_core::config::TrustedServerAppConfig; +use trusted_server_core::config_payload::VerifiedConfigBlob; +use trusted_server_core::ec::storage::{EcRuntime, EcStorageSelection}; +use trusted_server_core::settings::Settings; + +fn source(storage: &str) -> String { + format!( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "fixture" + password = "fixture-password" + [publisher] + domain = "publisher.example.com" + cookie_domain = ".publisher.example.com" + origin_url = "https://origin.example.com" + proxy_secret = "unit-test-proxy-secret" + [ec] + passphrase = "test-secret-key-32-bytes-minimum" + {storage} + "# + ) +} + +#[test] +fn every_runtime_resolves_only_its_supported_explicit_provider() { + let rows = [ + ("", ["none", "none", "none", "none"]), + ( + r#"ec_store = "exact-fixture-binding""#, + ["fastly", "none", "none", "none"], + ), + ( + "[ec.storage]\nstore = \"exact-fixture-binding\"", + ["fastly", "error", "error", "error"], + ), + ( + "[ec.storage]\nprovider = \"fastly\"\nstore = \"exact-fixture-binding\"", + ["fastly", "error", "error", "error"], + ), + ( + "[ec.storage]\nprovider = \"dynamodb\"\nregion = \"us-east-1\"\ntable = \"example-identity\"\nnamespace = \" exact \"", + ["error", "dynamodb", "error", "error"], + ), + ( + "[ec.storage]\nprovider = \"celld\"\nendpoint = \"http://127.0.0.1:7780\"\nstore = \"identity\"\n[ec.storage.access]\nmode = \"trusted-network\"", + ["error", "celld", "error", "error"], + ), + ]; + for (input, expected) in rows { + let settings = + Settings::from_toml(&source(input)).expect("should parse shape independent of runtime"); + for (runtime, expected) in [ + EcRuntime::Fastly, + EcRuntime::Native, + EcRuntime::Cloudflare, + EcRuntime::Spin, + ] + .into_iter() + .zip(expected) + { + let actual = match settings.ec.resolve_storage(runtime) { + Ok(None) => "none", + Ok(Some(EcStorageSelection::Fastly(store))) => { + assert_eq!(store, "exact-fixture-binding"); + "fastly" + } + Ok(Some(EcStorageSelection::DynamoDb(config))) => { + assert_eq!(config.namespace, " exact "); + "dynamodb" + } + Ok(Some(EcStorageSelection::Celld(_))) => "celld", + Err(_) => "error", + }; + assert_eq!( + actual, expected, + "should resolve runtime {runtime:?} for {input}" + ); + } + let value = serde_json::to_value(&settings).expect("should serialize input"); + let published: TrustedServerAppConfig = serde_json::from_value(value.clone()) + .expect("should preserve runtime-independent CLI input"); + let envelope = BlobEnvelope::new( + serde_json::to_value(published).expect("should publish typed input"), + "2026-01-01T00:00:00Z".to_owned(), + ); + assert_eq!( + envelope.data, value, + "should not inject defaults into published envelope" + ); + let encoded = serde_json::to_string(&envelope).expect("should encode existing envelope"); + VerifiedConfigBlob::parse(&encoded) + .expect("should reload and verify unchanged envelope format"); + let roundtrip = + Settings::from_json_value(value.clone()).expect("should reload published shape"); + assert_eq!( + serde_json::to_value(roundtrip).expect("should serialize roundtrip"), + value + ); + if !input.contains("provider") { + assert!( + value["ec"]["storage"].get("provider").is_none(), + "should not publish an inferred provider" + ); + } + } +} + +#[test] +fn ec_storage_published_shape_accepts_explicit_celld_without_credentials() { + let settings = Settings::from_toml(&source( + r#" + [ec.storage] + provider = "celld" + endpoint = "https://stores.example.com:7780" + store = "identity" + [ec.storage.access] + mode = "authenticated" + token_env = "FIXTURE_STORE_TOKEN" + "#, + )) + .expect("should accept pure celld config without reading a token"); + let value = serde_json::to_value(&settings).expect("should serialize publication input"); + assert_eq!(value["ec"]["storage"]["provider"], "celld"); + assert_eq!( + value["ec"]["storage"]["access"], + json!({"mode":"authenticated","token_env":"FIXTURE_STORE_TOKEN"}) + ); +} + +#[test] +fn absent_ec_storage_stays_absent_in_json() { + let settings = Settings::from_toml(&source("")).expect("should preserve unconfigured settings"); + let value = serde_json::to_value(settings).expect("should serialize settings"); + assert!( + value["ec"].get("storage").is_none(), + "should not inject provider defaults or null storage" + ); +} + +#[test] +fn ambiguous_incomplete_unknown_and_inappropriate_fields_are_rejected() { + for input in [ + "ec_store = \"same\"\n[ec.storage]\nstore = \"same\"", + "[ec.storage]", + "[ec.storage]\nstore = \"\"", + "[ec.storage]\nprovider = \"unknown\"\nstore = \"identity\"", + "[ec.storage]\nprovider = \"fastly\"\nstore = \"identity\"\nregion = \"us-east-1\"", + "[ec.storage]\nprovider = \"dynamodb\"\ntable = \"example-identity\"\nnamespace = \"fixture\"", + "[ec.storage]\nprovider = \"dynamodb\"\nregion = \"us-east-1\"\ntable = \"example-identity\"\nnamespace = \"\"", + "[ec.storage]\nprovider = \"celld\"\nendpoint = \"https://stores.example.com\"\nstore = \"identity\"", + "[ec.storage]\nprovider = \"celld\"\nendpoint = \"http://stores.example.com\"\nstore = \"identity\"\n[ec.storage.access]\nmode = \"authenticated\"\ntoken_env = \"FIXTURE_STORE_TOKEN\"", + ] { + let error = Settings::from_toml(&source(input)) + .expect_err("should reject invalid provider input without a runtime"); + assert!( + format!("{error:?}").contains("ec.storage"), + "should identify EC storage configuration" + ); + } +} + +#[test] +fn typed_local_endpoint_preserves_emulator_restrictions_and_redacts_errors() { + for endpoint in [ + "http://example.com:8000", + "https://127.0.0.1:8000", + "http://user:secret@127.0.0.1:8000", + "http://127.0.0.1:8000/private", + "http://127.0.0.1:8000/?secret=value", + "http://127.0.0.1:8000/#secret", + "http://127.0.0.1", + ] { + let input = format!( + "[ec.storage]\nprovider = \"dynamodb\"\nregion = \"us-east-1\"\ntable = \"example-identity\"\nnamespace = \"fixture\"\nlocal_endpoint = {endpoint:?}" + ); + let error = Settings::from_toml(&source(&input)) + .expect_err("should reject a nonlocal or ambiguous endpoint"); + let rendered = format!("{error:?}"); + assert!(rendered.contains("ec.storage.local_endpoint")); + assert!( + !rendered.contains(endpoint), + "should not echo endpoint values" + ); + } +} diff --git a/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml index c3d9ff53d8..14e3fecb10 100644 --- a/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml +++ b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml @@ -11,9 +11,11 @@ proxy_secret = "integration_proxy_secret" [ec] passphrase = "integration_ec_passphrase" -ec_store = "ec_identity_store" pull_sync_concurrency = 3 +[ec.storage] +store = "ec_identity_store" + [[ec.partners]] name = "Integration Test Partner" source_domain = "inttest.example.com" diff --git a/crates/trusted-server-integration-tests/tests/common/config.rs b/crates/trusted-server-integration-tests/tests/common/config.rs index db4e011c5e..be2aaf0481 100644 --- a/crates/trusted-server-integration-tests/tests/common/config.rs +++ b/crates/trusted-server-integration-tests/tests/common/config.rs @@ -9,7 +9,12 @@ const GENERATED_AT: &str = "2026-06-23T00:00:00Z"; const APP_CONFIG: &str = include_str!("../../fixtures/configs/trusted-server.integration.toml"); pub fn integration_app_config_envelope(origin_port: u16) -> TestResult { - app_config_envelope(origin_port, |_| Ok(())) + app_config_envelope(origin_port, |table| { + // The shared source names a Fastly binding. Native and Cloudflare + // fixtures intentionally retain their previous unconfigured graph. + table_mut(table, "ec")?.remove("storage"); + Ok(()) + }) } /// Builds the integration app-config envelope after `customize` edits the @@ -61,6 +66,7 @@ pub fn cloudflare_config_json(origin_port: u16) -> TestResult { /// through the orchestrator and telemetry without any outbound provider call. pub fn cloudflare_auction_config_json(origin_port: u16) -> TestResult { let envelope = app_config_envelope(origin_port, |table| { + table_mut(table, "ec")?.remove("storage"); let auction = table_mut(table, "auction")?; auction.insert("enabled".to_string(), toml::Value::Boolean(true)); auction.remove("providers"); @@ -91,6 +97,35 @@ fn cloudflare_binding_json(binding: &serde_json::Value) -> TestResult { #[cfg(test)] mod tests { + #[test] + fn native_and_cloudflare_fixtures_do_not_select_a_fastly_graph() { + let native = super::integration_app_config_envelope(8888) + .expect("should generate native fixture envelope"); + let envelope: edgezero_core::blob_envelope::BlobEnvelope = + serde_json::from_str(&native).expect("should parse native envelope"); + assert!( + envelope.data["ec"].get("storage").is_none(), + "should retain native unconfigured persistence" + ); + for binding in [ + super::cloudflare_config_json(8888), + super::cloudflare_auction_config_json(8888), + ] { + let binding: serde_json::Value = + serde_json::from_str(&binding.expect("should generate Cloudflare fixture")) + .expect("should parse Cloudflare binding"); + let encoded = binding[trusted_server_core::config_payload::CONFIG_BLOB_KEY] + .as_str() + .expect("should contain existing envelope"); + let envelope: edgezero_core::blob_envelope::BlobEnvelope = + serde_json::from_str(encoded).expect("should parse Cloudflare envelope"); + assert!( + envelope.data["ec"].get("storage").is_none(), + "should retain Cloudflare unconfigured persistence" + ); + } + } + const FASTLY_CONFIG: &str = include_str!("../../../../fastly.toml"); const VICEROY_TEMPLATE: &str = include_str!("../../fixtures/configs/viceroy-template.toml"); const LOGICAL_SECRET_STORE_ID: &str = "trusted_server_secrets"; diff --git a/docs/guide/native-ec-persistence.md b/docs/guide/native-ec-persistence.md index d1bf79d8c0..514a3dcce7 100644 --- a/docs/guide/native-ec-persistence.md +++ b/docs/guide/native-ec-persistence.md @@ -1,6 +1,6 @@ # Native EC persistence -The native Axum adapter can use a dedicated DynamoDB EC backend. Build with `aws-dynamodb-ec` and select it explicitly at startup. This is separate from ordinary byte KV and from immutable AppConfig or secret snapshots. A local database file on an ephemeral ECS volume is not a substitute for retained identity state. +The native Axum adapter can use dedicated DynamoDB or celld EC persistence. Build with `aws-dynamodb-ec` or `celld-ec` and select the provider explicitly in published `[ec.storage]` configuration. This is separate from ordinary byte KV and from immutable AppConfig or secret snapshots. A local database file on an ephemeral ECS volume is not a substitute for retained identity state. Persistence does not grant consent. Keep the shared EC consent, bot, browser-navigation and withdrawal rules intact. A request can be eligible for an auction but ineligible for an EC. Missing qualifying consent in a jurisdiction that requires it must still prevent EC issuance. Consent capture and CMP integration are separate work. @@ -37,23 +37,69 @@ docker build --platform linux/amd64 \ -t trusted-server-native-ec:local . ``` -This command is an operator example, not evidence of a qualified image. Rebuild the final runtime image from the new binary and record its source, dependency and image digests. A historical AppConfig/Secrets Manager image qualification does not cover EC persistence merely because environment variables were added. +This command is an operator example, not evidence of a qualified image. Rebuild the final runtime image from the new binary and record its source, dependency and image digests. A historical AppConfig/Secrets Manager image qualification does not cover EC persistence merely because a provider was added to published configuration. -Set these variables on the application container, not the AppConfig Agent: +Publish the non-secret selection in the existing Trusted Server app-config BlobEnvelope, through the same `ts config push`/AppConfig publication path used for other settings: -```sh -TS_EC_DYNAMODB_ENABLED=true -TS_EC_DYNAMODB_REGION=us-east-1 -TS_EC_DYNAMODB_TABLE=example-retained-ec -TS_EC_DYNAMODB_NAMESPACE=example-staging +```toml +[ec.storage] +provider = "dynamodb" +region = "us-east-1" +table = "example-retained-ec" +namespace = "example-staging" ``` -`TS_EC_DYNAMODB_ENABLED=true` selects the provider. `false` or absence leaves it off. Region, table and namespace are required when selected. Use the same region as the table. The TS region setting is explicit; do not rely on an unrelated `AWS_REGION` value to select the EC table region. Selected-provider preparation must succeed before readiness. Do not treat a selected-provider failure as permission to fall back to memory or emit an unbacked cookie. +Region, table and namespace are required. Preserve the exact existing table and namespace when migrating. Use the same region as the table; unrelated `AWS_REGION` values do not select the EC region. No provider or default binding is inferred on native. An ordinary KV declaration does not enable EC persistence. Selected-provider preparation must succeed before readiness. An error never permits fallback to memory, another provider or an unbacked cookie. + +This is a hard cutover from native environment-only selection. `TS_EC_DYNAMODB_ENABLED=true` now fails before readiness with migration guidance, even if new config is also present. Malformed enablement also fails. Absent or `false` does not select a backend; orphaned old binding variables are ignored, not another source. Prepare and publish the new config, then remove the old enablement/region/table/namespace/endpoint launch inputs. There is no fallback reader. + +Legacy `[ec] ec_store = "exact-binding"` remains supported on Fastly and inactive on other adapters. New Fastly config may use `[ec.storage] store = "exact-binding"`, with an omitted or explicit `fastly` provider. Both forms together are rejected. Cloudflare and Spin reject any explicit storage selection before routed application work. Shared parsing and publication do not resolve providers according to the CLI host. Missing storage stays absent when serialized, with no injected provider defaults. -The provider uses the standard AWS workload credential chain. In ECS, the application uses the task role. Keep credential values out of deployment inputs, logs, Terraform and container environment. The execution role still handles image pull, logs and task startup secrets, not application DynamoDB calls. Without a local-test selection, production preparation ignores configured endpoint overrides and uses the regional AWS endpoint. Local-only acceptance can set `TS_EC_DYNAMODB_LOCAL_ENDPOINT=http://127.0.0.1:8000`. This seam accepts only loopback HTTP with an explicit port and root path, bypasses the AWS config and credential chain, and uses dummy credentials. Never set it in a deployed task. In-process fixtures can also inject a client through the test preparation API. +Older binaries reject the new section. Deploy a compatible binary before publishing it; rollback requires an old-compatible config and matching binary together. A provider or namespace switch does not migrate identity data or withdrawal obligations. + +The provider uses the standard AWS workload credential chain. In ECS, the application uses the task role. Keep credential values out of deployment inputs, logs, Terraform and container environment. The execution role still handles image pull, logs and task startup secrets, not application DynamoDB calls. Without a local-test selection, production preparation ignores configured endpoint overrides and uses the regional AWS endpoint. Local-only acceptance can set `local_endpoint = "http://127.0.0.1:8000"` inside `[ec.storage]`. This seam accepts only loopback HTTP with an explicit port and root path, bypasses the AWS config and credential chain, and uses dummy credentials. Never set it in a deployed task. In-process fixtures can also inject a client through the test preparation API. The table is a single-region base table. Do not add Global Tables or route writes to different regions and assume the same CAS guarantees. A single namespace partition and generation counter can become hot; `PAY_PER_REQUEST` is not a throughput qualification. +## Celld conditional records + +Build the native adapter with `celld-ec` and publish an explicit binding: + +```toml +[ec.storage] +provider = "celld" +endpoint = "https://stores.example.com:7780" +store = "checkout-identity" + +[ec.storage.access] +mode = "authenticated" +token_env = "CHECKOUT_STORE_TOKEN" +``` + +The endpoint, service-owned alias and access policy use EdgeZero's pure celld settings. Do not publish a physical namespace or bearer token. The selected bootstrap variable is captured once during preparation; request paths do not reread it. Authenticated access requires HTTPS with normal certificate verification. Trusted-network access is an explicit alternative under the existing celld policy, never an authentication fallback. + +The provider requires the additive conditional-record protocol and checks both store and record version headers before readiness. Ordinary raw celld KV and the `celld-kv` feature do not supply the EC contract. Missing compiled provider, invalid credentials, unsupported protocol and unavailable service all fail closed. + +The typed client maps the unchanged EC trait onto body and opaque metadata, full-u64 server generations, atomic Add/Overwrite/IfGenerationMatch, exact physical existence, bounded strong prefix listing, TTL and hard deletion. A full list page is not proof of completeness. The server samples relative millisecond expiry inside the write transaction; finer positive client TTLs round upward. The core graph still owns every lifetime, tombstone and CAS retry. Transport does not replay uncertain mutations. Record cleanup must never reset the durable allocator. Raw KV keys and conditional record keys are separate datasets. + +Run the real disposable local-server proof explicitly with an existing tested celld binary: + +```sh +CELLD_EC_TEST_BINARY=/absolute/path/to/celld \ + cargo test --locked --target x86_64-unknown-linux-gnu \ + -p trusted-server-adapter-axum --features celld-ec \ + --test native_ec --test startup -- --include-ignored --test-threads=1 + +CELLD_EC_TEST_BINARY=/absolute/path/to/celld \ + cargo test --locked --target x86_64-unknown-linux-gnu \ + -p trusted-server-adapter-axum --features celld-ec,geo-cloudfront \ + --test native_acceptance \ + real_celld_envelope_drives_native_http_issuance_withdrawal_restart_and_outage \ + -- --ignored --test-threads=1 +``` + +Missing prerequisites fail instead of silently skipping this selected check. Fixtures own temporary policy/data and loopback listeners, not operator configuration. They exercise the real typed client and graph through native issuance, EID update, withdrawal, restart, outage and published-envelope readiness. The second check drives actual native HTTP issuance from the published envelope using a real origin and celld, synthetic protected-ingress geography and isolated secret references. It covers application recreation, server restart and outage without replacing the graph with a memory double. Tests are serialized to keep subprocess listener ownership separate. Synthetic local geography and GPC inputs are not genuine live-browser consent acceptance. Same-disk fixture restart is not replicated disaster recovery or production storage qualification. The server's separate fresh-node-cache LTX proof does not certify a production object-store deployment either. + ## Table and namespace ownership The [self-contained Terraform example](../../deploy/examples/native-ec-dynamodb/README.md) creates a retained table and attaches exact-table permissions to a supplied task role. It does not create or deploy an ECS service. diff --git a/docs/superpowers/specs/2026-11-03-ec-storage-provider-configuration-design.md b/docs/superpowers/specs/2026-11-03-ec-storage-provider-configuration-design.md new file mode 100644 index 0000000000..911154c24c --- /dev/null +++ b/docs/superpowers/specs/2026-11-03-ec-storage-provider-configuration-design.md @@ -0,0 +1,377 @@ +# EC storage provider configuration + +Status: Design decisions confirmed. This document specifies configuration and +adapter selection changes, not an implemented feature. Native environment +configuration gets a hard cutover, and supplying both configuration forms is +always an error. + +## Outcome + +Configure EC identity persistence through `trusted-server.toml` and its existing +published app-config envelope. Preserve existing Fastly configurations and the +behavior of deployments without configured persistence. Make provider selection +independent of the serving runtime, while rejecting combinations that have no +implementation. + +Fastly defaults to its own KV backend when a store is configured without an +explicit provider. Native, Cloudflare and Spin require explicit new provider +configuration to enable persistence. Merely running on a platform, declaring an +ordinary KV store, or supplying AWS credentials does not enable EC persistence. + +DynamoDB remains native-only in this change. Supporting another provider on +Fastly or Cloudflare later must not require a different configuration model or +changes to EC identity business logic. + +## Current behavior and constraints + +- `Ec.ec_store` in `crates/trusted-server-core/src/settings.rs` is an optional + Fastly store name. The example config sets it to `ec_identity_store`. +- Fastly's `maybe_identity_graph` and `require_identity_graph` in + `crates/trusted-server-adapter-fastly/src/main.rs` construct `FastlyEcKvStore` + directly from that name. The backend opens exactly that Fastly binding. +- Native startup loads verified Trusted Server settings, then separately calls + `ec_kv::prepare_from_env`. `TS_EC_DYNAMODB_*` variables select and configure + its dedicated EC backend. The provider requires `aws-dynamodb-ec`. +- Cloudflare and Spin do not currently wire an EC identity graph. Their ordinary + KV handles are not evidence of the conditional writes, generations and strong + reads required by `EcKvStore`. +- Native uses immutable startup config and secret snapshots, but identity rows + remain live. Changing how the provider is configured must not snapshot rows. + +Unconfigured persistence is not a new global EC-disable switch. Shared EC code +can generate an ID without a graph, while native publisher issuance currently +requires a graph. Preserve each adapter's existing unconfigured behavior rather +than using this migration to unify cookie issuance or route eligibility. + +Relevant contracts are documented in +[the native persistence guide](../../guide/native-ec-persistence.md) and +[the RC431 compatibility design](2026-10-20-edgezero-431-compatibility-design.md). +The EC backend operations live in +`crates/trusted-server-core/src/ec/kv_backend.rs`. + +## Configuration + +### Existing Fastly config remains valid + +```toml +[ec] +ec_store = "ec_identity_store" +``` + +On Fastly this continues to use the named Fastly KV binding. No new section, +provider field or environment variable is required. The name is not rewritten +or replaced with a manifest default. + +### New Fastly configuration + +```toml +[ec.storage] +store = "ec_identity_store" +``` + +On Fastly, omitting `provider` selects `fastly`. The fully explicit equivalent is: + +```toml +[ec.storage] +provider = "fastly" +store = "ec_identity_store" +``` + +Neither example supplies a default store name. An empty or incomplete storage +section is an error, not permission to enable `ec_identity_store` implicitly. + +### Native DynamoDB configuration + +```toml +[ec.storage] +provider = "dynamodb" +region = "us-east-1" +table = "example-retained-ec" +namespace = "example-production" +``` + +Region, table and namespace are required. Preserve current validation rules, +including exact namespace bytes. Do not derive the namespace from a task ID, +publisher domain or deployment run. Do not infer the provider from AWS settings, +field names, or an ordinary EdgeZero KV declaration. + +AWS credentials stay outside application config and use the standard SDK +credential chain, including an ECS task role. There are no access-key fields. +The existing `aws-dynamodb-ec` build feature is still required. + +For explicit local testing, the DynamoDB variant may also contain: + +```toml +local_endpoint = "http://127.0.0.1:8000" +``` + +This carries forward the existing emulator selection, not a general endpoint +override. Keep its current loopback HTTP, explicit-port and root-path validation, +and dummy credentials that bypass the ambient credential chain. Without it, +production uses the regional AWS endpoint and ignores endpoint overrides as it +does today. Do not place this field in production examples. + +### Native celld configuration + +```toml +[ec.storage] +provider = "celld" +endpoint = "https://stores.example.internal:7780" +store = "checkout-identity" + +[ec.storage.access] +mode = "authenticated" +token_env = "CHECKOUT_STORE_TOKEN" +``` + +The endpoint, exact service-owned alias and access policy use EdgeZero's pure +celld settings validation. No physical namespace or token value belongs in +application config. Capture the explicitly referenced bootstrap credential once +when preparing the backend. Authenticated mode requires HTTPS with normal +certificate verification. Trusted-network mode is an explicit alternative with +the existing celld rules, not a fallback after authentication fails. + +The native build must include `celld-ec`, which enables the separate typed +conditional-record client in `edgezero-store-celld`. An ordinary KV declaration +or `celld-kv` feature does not select an identity backend. Native preparation +requires the additive record protocol, not just the original raw-value API. + +Celld implements the same `EcKvStore` contract, including opaque metadata, +server-assigned u64 generations, Add/Overwrite/IfGenerationMatch, physical exact +existence, strong bounded prefix listing, expiry and deletion. The server owns +atomic preconditions and durable generation allocation. Reuse core graph and +consent logic unchanged. Do not emulate conditional writes with client GET/PUT, +redirect credentials, replay ambiguous mutations, or reset an allocator during +ordinary identity cleanup. + +Celld is initially native-only, like DynamoDB. This does not claim a WASM +transport or add a placeholder implementation for another serving runtime. + +## Resolution rules + +Resolve configuration against an explicit serving-runtime identity, not the +host that runs the CLI or the Rust compilation target alone. + +| Input | Fastly | Native Axum | Cloudflare / Spin | +| ---------------------------------------------- | --------------------------- | --------------------------------- | --------------------------------- | +| Neither `ec_store` nor `storage` | No graph, existing behavior | No graph, existing behavior | Existing behavior, no graph | +| Legacy `ec_store` only | Named Fastly graph | Legacy field remains inactive | Legacy field remains inactive | +| `storage.store`, provider omitted | Fastly, named store | Error, explicit provider required | Error, explicit provider required | +| Explicit `provider = "fastly"`, valid store | Fastly graph | Unsupported-provider error | Unsupported-provider error | +| Explicit `provider = "dynamodb"`, valid fields | Unsupported-provider error | DynamoDB graph, feature required | Unsupported-provider error | +| Explicit `provider = "celld"`, valid fields | Unsupported-provider error | Celld graph, feature required | Unsupported-provider error | +| Both `ec_store` and `storage` | Configuration error | Configuration error | Configuration error | + +Additional rules: + +- Any simultaneous legacy field and new section is ambiguous and rejected, + even if they name the same Fastly store. Do not invent precedence or merge + fields across the two forms. Switching providers means removing `ec_store`. +- Reject unknown provider names, unknown fields, inappropriate variant fields, + missing required fields and empty store names with field-oriented errors. +- Do not select the first available store or fall back after a selected-provider + error. An unavailable configured store is not an unconfigured deployment. +- The legacy field remains inactive on non-Fastly adapters because that is its + existing behavior. Explicit new configuration must never be silently ignored. + This distinction keeps shared legacy config files usable during migration. +- Absence must survive serialization as absence. Do not publish an inferred + provider, `storage = null`, or a generated default table into old config blobs. + +The initial support matrix is deliberately small. Cloudflare and Spin have no +supported EC persistence provider in this change. New explicit storage config +there fails; deployments without it continue to work as before. Do not add +placeholder `cloudflare` or generic `kv` variants that cannot meet the contract. + +## Ownership and implementation boundaries + +Use one shared configuration resolver to produce either no persistence or a +validated provider-specific selection. A typed resolved enum should distinguish +Fastly binding configuration, DynamoDB configuration and celld configuration. Parsing compatibility +and the omitted-provider rule must not become several loosely synchronized +booleans and optional fields used throughout the request path. + +Shared settings, validation and resolution must not depend on Fastly or AWS +SDKs. Adapters own provider construction and platform resources. A future +provider implementation may be usable on multiple runtimes; a runtime name is +not a permanent restriction on the provider model. No dynamic plugin registry +or universal byte-KV wrapper is needed for these implementations. + +```mermaid +flowchart TD + A[Load and verify existing app-config envelope] --> B[Validate storage shape] + B --> C[Resolve against serving runtime] + C --> D{Persistence configured?} + D -->|No| E[Preserve existing unconfigured behavior] + D -->|Yes| F{Provider supported by this runtime and build?} + F -->|No| X[Configuration error before routed application work] + F -->|Yes| G[Adapter prepares selected backend] + G --> H[Supply live store to shared EC graph] + G -->|Preparation fails| X +``` + +Native replaces `prepare_from_env()` with typed configuration input and awaits +provider preparation before readiness. Keep network I/O asynchronous, existing +bounds and sanitized errors, table validation, allocator continuity, logical +expiry, conditional writes and withdrawal semantics unchanged. + +Fastly must use the resolved selection everywhere a graph is constructed, +including organic routes, auctions, identify, batch sync, finalization and +post-send pull sync. Do not update only one helper and leave other consumers +reading `ec.ec_store` directly. Preserve existing Fastly store-opening timing +and health/probe exceptions; this spec does not introduce remote readiness +checks or alter request/egress ownership. + +Cloudflare and Spin must validate explicit storage selection during application +initialization, before routed application work, even though no new backend is +added there. A successful schema parse alone is not support for a provider. + +Keep two validation stages distinct: + +1. Shared TOML/JSON shape validation can run without a network or serving runtime. +2. Runtime-aware resolution and provider preparation run in the selected adapter. + A target-aware CLI validation path may reject unsupported combinations early, + but a native CLI must not reject Fastly config merely because it runs natively. + +`ts config push` continues to publish the existing BlobEnvelope. AppConfig +continues to carry that same envelope where selected. Do not add another file +loader, environment overlay or publication format. EdgeZero still owns the +framework lifecycle and config/secret provisioning. This change does not claim +that its ordinary KV interface supplies the EC backend contract. + +## Migration + +### Fastly compatibility + +Keep legacy `ec.ec_store` as a supported shorthand, without a removal deadline +in this change. New examples use `[ec.storage]`, but existing operator files and +already published envelopes need no rewrite. Migrate repository-owned active +examples and fixtures together, retaining dedicated legacy-compatibility tests. +Historical specs are records, not active config consumers to rewrite. + +Older binaries use `deny_unknown_fields` on EC settings and cannot read the new +section. Deploy a compatible binary before publishing new-shaped config. For a +rollback, restore an old-compatible config and the matching binary together. +The migration must preserve the exact binding, table and namespace, not move +identity data or reset the generation allocator. + +### Native environment-variable hard cutover + +Migrate repository-owned launchers and deployment outputs in the same change, +then remove the old selection path. There is no compatibility release or +fallback reader that prepares a backend from legacy environment values. + +- Move the four non-secret selection/binding values into `ec.storage` in the + published envelope. Remove `TS_EC_DYNAMODB_ENABLED`, `REGION`, `TABLE` and + `NAMESPACE` from application deployment inputs after publication is prepared. +- Replace Terraform's `runtime_environment` output in + `deploy/examples/native-ec-dynamodb/outputs.tf` with a documented `ec_storage` + configuration object for publication. This is an output-contract change; + inspect downstream consumers before removal. It does not apply or provision AWS. +- Update `scripts/test-native-ec-image.mjs` to place provider config and + `local_endpoint` in its fixture envelope instead of the app environment. +- Replace the app's `TS_EC_DYNAMODB_LOCAL_ENDPOINT` reader with the typed local + field. A manual test runner may retain a test-only endpoint input, clearly + distinguished from production application configuration. +- Update selection tests, the native persistence guide and the Terraform README. + Inventory deployment automation, including untracked local work, without + modifying unrelated work or operator-owned `trusted-server.toml` files. + +Do not silently ignore an old active selection and start without a graph. A +native process still receiving an enabled legacy selection fails before +readiness with migration guidance, including when new +config is also present. Malformed legacy enablement remains an error. Absent or +`false` enablement does not select a backend; orphaned old binding values are +not an alternate configuration source. Errors must not echo record contents, +credentials or endpoint values. + +This migration check detects an obsolete launch contract; it never prepares +DynamoDB from environment values. + +## Persistence invariants and non-goals + +For configured persistence, new identity issuance must still await a confirmed +write. Failed or uncertain writes must not become successful issuance through +a fallback store or a retry of a different backend. Consent, browser and route +gates remain unchanged. All graph consumers in an application use the same +resolved backend. + +Changing provider or binding configuration is not data migration. This change +provides no copying, dual writes, cross-provider failover, namespace rotation or +automatic recovery plan for identities left in another store. A real switch +requires a separate continuity and withdrawal plan. Even changing only a +DynamoDB namespace changes the identity dataset. + +Other non-goals are DynamoDB/celld EC on WASM runtimes, Cloudflare EC backend creation, +ordinary KV redesign, new native EC routes, pull-sync parity, hot reload, +credentials in TOML, table provisioning and changes to stored record formats. + +## Acceptance checks + +- Table-driven resolution tests cover every row in the matrix and all serving + runtimes. Test malformed, empty and unsupported selections and duplicate forms. +- TOML and JSON round trips preserve both legacy config and absent storage. + Config push and envelope reload preserve explicit new config without injecting + defaults or resolving providers according to the CLI host. +- An unchanged legacy Fastly fixture uses the exact previous binding. New + omitted-provider and explicit-Fastly fixtures use that same binding. No-store + fixtures retain their existing graph availability and cookie behavior. +- All Fastly graph consumers use the resolved selection. Unsupported explicit + selections fail before route work, with health/probe exceptions unchanged. +- Native feature-off builds accept unconfigured storage and reject selected + DynamoDB before readiness. Feature-on tests prepare from envelope data, not + legacy environment inputs. Obsolete active launch inputs cannot silently + disable persistence or override the new config. +- Emulator tests use the new local field and retain endpoint-validation and + dummy-credential isolation checks. Reuse existing backend tests for CAS, + expiry, deletion/recreation and withdrawal where inputs remain unchanged. +- Native end-to-end fixtures prove issuance, persistence across application + recreation, withdrawal and failed-write behavior through the new envelope path. +- Cloudflare and Spin compile without AWS dependencies and reject explicit + unsupported new storage config. Shared legacy configs remain accepted. +- Inspect a Terraform config output and its documented publication path. Search + active application launch inputs and docs for obsolete variables. Run focused + formatting and affected crate checks during implementation; deployment or live + IAM acceptance requires separate authorization. + +## Contradiction review + +### Default provider versus implicit enablement + +Resolved by separating the two. Fastly is the default provider only when a store +is configured. With neither configuration form, nothing new is enabled. There +is no invented `ec_identity_store` binding and no global EC-disable behavior. + +### Migrating configs versus not requiring existing users to change them + +Resolved for Fastly by retaining `ec.ec_store` as supported input while moving +repository examples to the new section. Native env-based deployments necessarily +need an input migration to remove environment-only configuration. The confirmed +rollout is a hard cutover with coordinated config and launch-input changes, +without a staged compatibility release. + +### Independent provider choice versus native-only DynamoDB + +Resolved by separating configuration from runtime support. The schema and shared +resolver allow explicit provider choice; the initial implementations still only +support Fastly KV on Fastly and DynamoDB on native. Cross-provider use on Fastly +or Cloudflare is an extension goal, not delivered behavior in this change. + +### Preserving legacy input versus rejecting unsupported providers + +Legacy `ec_store` was inactive on non-Fastly adapters and remains so. A new +explicit section is an intentional selection and must fail if unsupported. This +asymmetry is necessary to preserve existing shared configs without pretending +that a newly requested backend works. + +### Switching providers versus keeping identities + +Not solved by configuration. Supporting a different provider later does not +make switching an existing deployment safe. Data continuity and withdrawal +obligations require separate migration work. + +## Confirmed review decisions + +1. Native environment configuration gets a hard cutover. Migrate config and + launch inputs together; do not retain an environment-backed fallback. +2. Supplying both `ec.ec_store` and `[ec.storage]` is always a configuration + error, even when equivalent. There are no precedence or merge rules. diff --git a/scripts/test-native-ec-image.mjs b/scripts/test-native-ec-image.mjs index 292285b824..61d5a20313 100644 --- a/scripts/test-native-ec-image.mjs +++ b/scripts/test-native-ec-image.mjs @@ -426,6 +426,13 @@ async function launch(label, originEndpoint, proxyEndpoint) { const fixture = { ec: { passphrase: 'ec_key', + storage: { + provider: 'dynamodb', + region: 'us-east-1', + table, + namespace, + local_endpoint: proxyEndpoint, + }, partners: [ { name: 'synthetic fixture', @@ -466,11 +473,6 @@ async function launch(label, originEndpoint, proxyEndpoint) { AWS_EC2_METADATA_DISABLED: 'true', AWS_REGION: 'us-east-1', AWS_DEFAULT_REGION: 'us-east-1', - TS_EC_DYNAMODB_ENABLED: 'true', - TS_EC_DYNAMODB_REGION: 'us-east-1', - TS_EC_DYNAMODB_TABLE: table, - TS_EC_DYNAMODB_NAMESPACE: namespace, - TS_EC_DYNAMODB_LOCAL_ENDPOINT: proxyEndpoint, TS_NATIVE_GEO_PROFILE_FILE: `${root}/native-geo.toml`, EDGEZERO__ADAPTER__CONFIG_DIR: root, EDGEZERO__ADAPTER__DATA_DIR: opt.binary ? dir : '/data', diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 066ad659e3..db3dcb2b46 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -75,16 +75,19 @@ proxy_secret = "publisher_proxy_secret" [ec] # Secret Store key for the secret used to derive EC identifiers. passphrase = "ec_passphrase" -# KV store that persists EC identity state. This is the physical store name -# bound per adapter (e.g. `ec_identity_store` in fastly.toml); edgezero.toml's -# logical KV id is `trusted_server_kv`. -ec_store = "ec_identity_store" # Max concurrent partner pull-sync requests. pull_sync_concurrency = 3 # Optional cluster-heuristic tuning (defaults shown): # cluster_trust_threshold = 10 # entries with cluster_size <= this are individual users # cluster_recheck_secs = 3600 # re-evaluate cluster_size after this many seconds +# Fastly persistence uses this exact physical binding. Provider omission is +# Fastly-only; native providers require explicit configuration. An ordinary +# logical KV declaration does not select EC storage. Legacy ec.ec_store remains +# supported on Fastly, but cannot be combined with this section. +[ec.storage] +store = "ec_identity_store" + # Example partner configuration. Provision referenced keys in # trusted_server_secrets before validating/pushing. # [[ec.partners]] From ad00c404bf9d50ea7d79066d8de07f494c0a760a Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 12 Nov 2026 06:22:22 -0600 Subject: [PATCH 2/3] Select runtime-appropriate storage in local acceptance fixtures The shared CLI template deliberately selects Fastly storage. Leave that graph intact in Fastly fixtures and remove only its known placeholder for Axum, Cloudflare and Spin smokes before typed validation and export. Give the EOF fixture its serving adapter and keep non-Fastly graphs unconfigured. Preserve production provider validation and all negative startup/body/timeout assertions. Native CLI export now reaches prepared startup without an unsupported Fastly selection. --- .../tests/html_eof_runtime.rs | 9 ++++++- .../tests/html_eof_runtime/config.rs | 25 +++++++++++++++++-- scripts/smoke-axum.sh | 2 +- scripts/smoke-cloudflare.sh | 2 +- scripts/smoke-common.sh | 14 +++++++++++ scripts/smoke-fastly.sh | 2 +- scripts/smoke-spin.sh | 2 +- 7 files changed, 49 insertions(+), 7 deletions(-) diff --git a/crates/trusted-server-integration-tests/tests/html_eof_runtime.rs b/crates/trusted-server-integration-tests/tests/html_eof_runtime.rs index ad42208425..4bac1bd655 100644 --- a/crates/trusted-server-integration-tests/tests/html_eof_runtime.rs +++ b/crates/trusted-server-integration-tests/tests/html_eof_runtime.rs @@ -667,7 +667,14 @@ fn actual_runtime() -> HarnessResult<()> { harness.cancellation.check()?; let port = available_port()?; // Auction controls deliberately use an unreachable bidder and verify empty-bid assembly. - let config = config::fixture_config(&template, case, origin.port, available_port()?, port)?; + let config = config::fixture_config( + &template, + &adapter, + case, + origin.port, + available_port()?, + port, + )?; fs::write(harness.workspace.join("trusted-server.toml"), config) .change_context(HarnessError::Io)?; fs::write(harness.workspace.join("fastly.toml"), "manifest_version = 3\nname = 'example-finalization'\nlanguage = 'rust'\n[local_server]\n").change_context(HarnessError::Io)?; diff --git a/crates/trusted-server-integration-tests/tests/html_eof_runtime/config.rs b/crates/trusted-server-integration-tests/tests/html_eof_runtime/config.rs index 2fc54a7ffe..e775ef165f 100644 --- a/crates/trusted-server-integration-tests/tests/html_eof_runtime/config.rs +++ b/crates/trusted-server-integration-tests/tests/html_eof_runtime/config.rs @@ -56,12 +56,18 @@ fn table<'a>( pub(super) fn fixture_config( template: &str, + adapter: &str, case: &str, origin: u16, bid: u16, public_port: u16, ) -> HarnessResult { let mut value = parse(template)?; + if adapter != "fastly" { + // The shared template selects Fastly's graph. Other EOF fixtures + // intentionally exercise their existing unconfigured EC path. + table(&mut value, &["ec"])?.remove("storage"); + } for (key, replacement) in [ ("domain", "127.0.0.1".to_owned()), ("cookie_domain", "127.0.0.1".to_owned()), @@ -263,7 +269,7 @@ mod tests { #[test] fn fixture_config_is_loopback_only_with_explicit_permission() { for case in ["disabled", "success", "hung", "auction", "streaming"] { - let config = fixture_config(CONFIG, case, 12345, 12346, 12347) + let config = fixture_config(CONFIG, "axum", case, 12345, 12346, 12347) .expect("should generate isolated configuration"); let parsed: toml::Value = toml::from_str(&config).expect("should parse configuration"); assert_eq!( @@ -294,10 +300,25 @@ mod tests { } } + #[test] + fn fixture_storage_matches_the_serving_runtime() { + for adapter in ["axum", "cloudflare", "spin", "fastly"] { + let config = fixture_config(CONFIG, adapter, "success", 12345, 12346, 12347) + .expect("should generate the runtime-specific fixture"); + let value = parse(&config).expect("should parse the fixture"); + assert_eq!( + value["ec"].get("storage").is_some(), + adapter == "fastly", + "should retain the Fastly graph only for its serving runtime" + ); + } + } + #[test] fn fixture_config_rejects_missing_required_template_fields() { let result = fixture_config( "[publisher]\ndomain = 'example.com'", + "axum", "success", 12345, 12346, @@ -308,7 +329,7 @@ mod tests { #[test] fn fixture_validation_rejects_external_endpoints() { - let config = fixture_config(CONFIG, "auction", 12345, 12346, 12347) + let config = fixture_config(CONFIG, "axum", "auction", 12345, 12346, 12347) .expect("should generate auction configuration"); let mut value = parse(&config).expect("should parse generated configuration"); let provider = table(&mut value, &["auction", "providers"]) diff --git a/scripts/smoke-axum.sh b/scripts/smoke-axum.sh index 3256fbf2c2..0f3a747b19 100755 --- a/scripts/smoke-axum.sh +++ b/scripts/smoke-axum.sh @@ -58,7 +58,7 @@ fi smoke_start_origin "$WORKSPACE" "$ORIGIN_PORT" ORIGIN_PID=$SMOKE_ORIGIN_PID -smoke_initialize_config "$REPO_ROOT" "$WORKSPACE" "$ORIGIN_PORT" "$((BASE_PORT + 4))" +smoke_initialize_config "$REPO_ROOT" "$WORKSPACE" "$ORIGIN_PORT" "$((BASE_PORT + 4))" "axum" sed \ -e "s|= \"crates/|= \"$REPO_ROOT/crates/|g" \ -e "s|manifest = \"fastly.toml\"|manifest = \"$REPO_ROOT/fastly.toml\"|" \ diff --git a/scripts/smoke-cloudflare.sh b/scripts/smoke-cloudflare.sh index 867504397b..5ce28a6b2c 100755 --- a/scripts/smoke-cloudflare.sh +++ b/scripts/smoke-cloudflare.sh @@ -46,7 +46,7 @@ fi smoke_start_origin "$WORKSPACE" "$ORIGIN_PORT" ORIGIN_PID=$SMOKE_ORIGIN_PID # The positive case uses the final listener in the case sequence. -smoke_initialize_config "$REPO_ROOT" "$WORKSPACE" "$ORIGIN_PORT" "$POSITIVE_PORT" +smoke_initialize_config "$REPO_ROOT" "$WORKSPACE" "$ORIGIN_PORT" "$POSITIVE_PORT" "cloudflare" CLOUDFLARE_WORK="$WORKSPACE/cloudflare" mkdir -p "$CLOUDFLARE_WORK" diff --git a/scripts/smoke-common.sh b/scripts/smoke-common.sh index 2e69ab0e18..1b5e1caf5f 100644 --- a/scripts/smoke-common.sh +++ b/scripts/smoke-common.sh @@ -183,8 +183,22 @@ smoke_initialize_config() { local workspace="$2" local origin_port="$3" local listener_port="$4" + local adapter="$5" SMOKE_APP_CONFIG="$workspace/trusted-server.toml" "$SMOKE_TS_BIN" config init --app-config "$SMOKE_APP_CONFIG" + if [ "$adapter" != "fastly" ]; then + smoke_require_command node + # The general template selects Fastly storage. Other runtime smokes + # need no EC provider. Remove only that known template placeholder. + node --input-type=module - "$SMOKE_APP_CONFIG" <<'JS' +import { readFileSync, writeFileSync } from 'node:fs' +const file = process.argv[2] +const source = readFileSync(file, 'utf8') +const fastlyStorage = '[ec.storage]\nstore = "ec_identity_store"\n' +if (!source.includes(fastlyStorage)) throw new Error('Expected the Fastly storage template block') +writeFileSync(file, source.replace(fastlyStorage, '')) +JS + fi export TRUSTED_SERVER__PUBLISHER__DOMAIN="127.0.0.1" export TRUSTED_SERVER__PUBLISHER__COOKIE_DOMAIN="127.0.0.1" export TRUSTED_SERVER__PUBLISHER__ORIGIN_URL="http://127.0.0.1:$origin_port" diff --git a/scripts/smoke-fastly.sh b/scripts/smoke-fastly.sh index 32de05bee7..f1148cb9e5 100755 --- a/scripts/smoke-fastly.sh +++ b/scripts/smoke-fastly.sh @@ -69,7 +69,7 @@ fi smoke_start_origin "$WORKSPACE" "$ORIGIN_PORT" ORIGIN_PID=$SMOKE_ORIGIN_PID # The positive case uses the final listener in the case sequence. -smoke_initialize_config "$FASTLY_PROJECT" "$WORKSPACE" "$ORIGIN_PORT" "$POSITIVE_PORT" +smoke_initialize_config "$FASTLY_PROJECT" "$WORKSPACE" "$ORIGIN_PORT" "$POSITIVE_PORT" "fastly" run_case() { local case_name="$1" diff --git a/scripts/smoke-spin.sh b/scripts/smoke-spin.sh index b12441aa2a..dbbc4a0aa5 100755 --- a/scripts/smoke-spin.sh +++ b/scripts/smoke-spin.sh @@ -44,7 +44,7 @@ fi smoke_start_origin "$WORKSPACE" "$ORIGIN_PORT" ORIGIN_PID=$SMOKE_ORIGIN_PID # The positive case uses the final listener in the case sequence. -smoke_initialize_config "$REPO_ROOT" "$WORKSPACE" "$ORIGIN_PORT" "$POSITIVE_PORT" +smoke_initialize_config "$REPO_ROOT" "$WORKSPACE" "$ORIGIN_PORT" "$POSITIVE_PORT" "spin" SPIN_WORK="$WORKSPACE/spin" mkdir -p "$SPIN_WORK" From 83b5bd027db10bf938cb9a4567a47f1422e2f8ed Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 12 Nov 2026 06:36:49 -0600 Subject: [PATCH 3/3] Exercise the runtime-specific smoke helper in CLI store tests Pass the serving adapter through the existing helper caller, keep both listener-port cases, and assert that Axum/Cloudflare/Spin omit Fastly storage while Fastly retains it. --- .../tests/config_store_defaults.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-cli/tests/config_store_defaults.rs b/crates/trusted-server-cli/tests/config_store_defaults.rs index 4b5249338e..318edce38c 100644 --- a/crates/trusted-server-cli/tests/config_store_defaults.rs +++ b/crates/trusted-server-cli/tests/config_store_defaults.rs @@ -122,7 +122,13 @@ fn development_overlay_replaces_the_template_public_origin() { #[cfg(unix)] #[test] fn smoke_setup_replaces_the_template_with_the_selected_listener() { - for listener_port in ["18894", "43123"] { + for (listener_port, adapter) in [ + ("18894", "axum"), + ("43123", "axum"), + ("43123", "cloudflare"), + ("43123", "spin"), + ("43123", "fastly"), + ] { let project = project(); fs::remove_file(project.path().join("trusted-server.toml")) .expect("should let smoke setup initialize the template"); @@ -131,10 +137,10 @@ fn smoke_setup_replaces_the_template_with_the_selected_listener() { concat!( include_str!("../../../scripts/smoke-common.sh"), "\nSMOKE_TS_BIN=\"$TS_BIN\"\n", - "smoke_initialize_config \"$PWD\" \"$PWD\" 19999 \"$1\"\n", + "smoke_initialize_config \"$PWD\" \"$PWD\" 19999 \"$1\" \"$2\"\n", "\"$TS_BIN\" config push --adapter axum --local --yes --no-diff\n", ), - &[listener_port], + &[listener_port, adapter], ); assert_eq!( config["publisher"]["public_origin"], @@ -145,6 +151,11 @@ fn smoke_setup_replaces_the_template_with_the_selected_listener() { config["publisher"]["origin_url"], "http://127.0.0.1:19999", "should keep the smoke upstream independent of the listener" ); + assert_eq!( + config["ec"].get("storage").is_some(), + adapter == "fastly", + "should retain the Fastly storage template only for its serving runtime" + ); } }