diff --git a/.cargo/config.toml b/.cargo/config.toml index aea0df2d59..0c81704c8b 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -28,8 +28,8 @@ test_details = "test --target aarch64-apple-darwin" # (native) are simply not listed. build-fastly = "build -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" check-fastly = "check -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" -clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" -test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" +clippy-fastly = "clippy -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-identity-providers -p trusted-server-js -p trusted-server-openrtb --all-targets --all-features --target wasm32-wasip1 -- -D warnings" +test-fastly = "test -p trusted-server-core -p trusted-server-adapter-fastly -p trusted-server-identity-providers -p trusted-server-js -p trusted-server-openrtb --target wasm32-wasip1" # Feature-on counterpart of `test-fastly`. `test-fastly` does NOT pass # --all-features, so without this the reusable-sandbox code is linted by # `clippy-fastly` but its tests never execute. diff --git a/AGENTS.md b/AGENTS.md index bf31f54858..25ff9508da 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,6 +21,7 @@ crates/ trusted-server-adapter-cloudflare/ # Cloudflare Workers entry point (wasm32-unknown-unknown binary) trusted-server-adapter-spin/ # Fermyon Spin entry point (wasm32-wasip1 component) trusted-server-cli/ # Host-target `ts` operator CLI + trusted-server-identity-providers/ # Server-side identity provider kinds (used by the Fastly adapter) trusted-server-js/ # TypeScript/JS build — per-integration IIFE bundles lib/ # TS source, Vitest tests, esbuild pipeline ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index dc8a24e872..38862aa663 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,10 +22,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 measured from dispatch, including delayed headers and trickling bodies on all adapters. Complete available responses survive a slow publisher origin; unfinished providers time out and mediation is skipped after budget exhaustion. - Provider bids that are already complete at the final nonblocking snapshot remain + Provider bids that are already complete at the final snapshot remain eligible even after `tmax`; mediation starts only with remaining budget and rejects responses collected after its cutoff. Fastly abandons unfinished requests; Axum, Cloudflare, and Spin cancel owned work. + On Fastly the final snapshot may wait up to 1 ms per partially received body so + the host can surface bytes that already arrived, and reads on into the body of + a response whose headers arrived by the cutoff instead of abandoning it. +- Added server-side identity providers (`[[ec.identity_providers]]`, Fastly only). Enabled providers look up a visitor's IDs on publisher navigations within `ec.identity_budget_ms` (1–1000, default 200), or after the response within `ec.identity_after_response_budget_ms` (1–5000, default 2000) for `collect = "after_response"`. A provider owns its `sources`: batch sync rejects them with the new `provider_owned` reason, pull sync skips them, and browser IDs never fill them. Set `tcf_vendor_id` to require TCF vendor consent for lookups and for every ID of the provider's sources in bidstream, `x-ts-eids` and `/identify`, browser-sent IDs included; a disabled provider keeps that gate. See [Identity providers](docs/guide/configuration.md#identity-providers). +- Batch sync reports a new `row_full` item reason when a mapping would push the EC row past its new 16 KiB size limit, and continues with the remaining groups. +- An EC row over 16 KiB written by an earlier release loses its largest partner records on its next write that adds an ID, so the added ID still lands. +- **Breaking:** An EC is now minted, and the EC row written, only on a top-level publisher navigation: a `GET` with `Sec-Fetch-Dest: document`, `Sec-Fetch-Mode: navigate` and no `prefetch` in `Sec-Purpose`. Clients that send no Fetch Metadata get no EC, a navigation answered with a 301, 302, 303, 307 or 308 redirect mints none, integration proxies no longer mint one, and prefetches and prerenders never mint or write. Each navigation makes at most one conditional page-load write; a consent withdrawal's tombstone is still written on any request that carries the EC cookie. +- **Breaking:** Browser EIDs from the `/auction` request body are no longer stored, and `/auction` and `/_ts/page-bids` no longer write browser EIDs from `ts-eids` or `sharedId`. Those cookies are ingested on the next top-level navigation. +- **Breaking:** An EC minted on the current navigation is no longer sent as `user.id` to bidders, that navigation's auction does not carry the browser's `ts-eids`, and identity providers are not called for it, until a later navigation finds its row. Identity providers are also not called for an EC cookie whose row does not exist. +- This release registers no identity provider kind, so `ts config push` rejects an enabled `[[ec.identity_providers]]` entry and every runtime refuses to start with one; configure providers with `enabled = false` until a kind ships. +- A consent withdrawal keeps the `ts-ec` cookie while its tombstone cannot be written (after bounded CAS retries, or when the row cannot be read), so the next request carrying the cookie retries. Previously the cookie was expired first. +- **Breaking:** Axum, Cloudflare and Spin refuse to start when an enabled identity provider is configured, and Fastly refuses to start when an enabled provider names a `kind` this build does not include. +- **Breaking (rollback):** Stored provider UIDs may be up to 1024 bytes (browser, batch and pull ingest stay at 512). A binary from before this release cannot read an EC row holding a longer UID, so roll back only after such rows expire or are rewritten. An older binary also ignores the full-fidelity `eids` and the 30-day provider `opt_outs` and drops them when it rewrites a row (for example in batch or pull sync), so opted-out visitors are looked up again after rolling forward. A config blob that sets `ec.identity_providers` or a non-default identity budget cannot be loaded by an older binary; restore the defaults and re-push before rolling back. - Explicit TS Console activation with `?ts_console=1` now sets a 30-minute diagnostics cookie instead of a browser-session cookie, independently of the mobile trace flag. Ordinary requests do not refresh it; end and explicitly enable diagnostics again to adopt the lifetime for an existing session. - Publisher documents with diagnostics active now use private, no-store responses and strip conditional and range request headers before fetching the origin. An unexpected origin `304` becomes a private `502` because it cannot supply an instrumented document, including when mobile tracing is disabled. - **Breaking:** `x-ts-version` now reports the deployed git version — the release tag, else the branch, else the first 6 characters of the commit — compiled in from the build-time `TRUSTED_SERVER__GIT_VERSION` (set by the deploy pipeline) or local git, and is sent by every adapter, including on the Fastly `GET /health` probe. The Fastly service version it previously carried is now `x-ts-fastly-version`; update dashboards, monitors, and scripts that read `x-ts-version` as the Fastly version number. Builds with neither the override nor git omit the header. diff --git a/Cargo.lock b/Cargo.lock index 70316712ac..d9cf35f9f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6604,6 +6604,7 @@ dependencies = [ "serde_json", "toml", "trusted-server-core", + "trusted-server-identity-providers", "url", "urlencoding", "web-time", @@ -6737,6 +6738,15 @@ dependencies = [ "web-time", ] +[[package]] +name = "trusted-server-identity-providers" +version = "0.1.0" +dependencies = [ + "error-stack", + "toml", + "trusted-server-core", +] + [[package]] name = "trusted-server-integration-tests" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 139cf4f610..f431012308 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crates/trusted-server-adapter-spin", "crates/trusted-server-cli", "crates/trusted-server-core", + "crates/trusted-server-identity-providers", "crates/trusted-server-integration-tests", "crates/trusted-server-js", "crates/trusted-server-openrtb", @@ -121,6 +122,7 @@ toml_edit = "0.23.10" tower = "0.4" tracing = "0.1" trusted-server-core = { path = "crates/trusted-server-core" } +trusted-server-identity-providers = { path = "crates/trusted-server-identity-providers" } trusted-server-js = { path = "crates/trusted-server-js" } trusted-server-openrtb = { path = "crates/trusted-server-openrtb" } url = "2.5.8" diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 0b1858e208..585a9d24fc 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -91,6 +91,7 @@ fn build_state_with_services( settings: Settings, services: Option, ) -> Result, Report> { + trusted_server_core::ec::provider::reject_enabled_providers(&settings.ec, "Axum")?; let plan = Arc::new(compile_auction_plan(&settings)?); plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Axum)?; let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; @@ -224,15 +225,12 @@ async fn dispatch_fallback( } if state.registry.has_route(&method, &path) { - let mut ec_context = EcContext::default(); return state .registry .handle_proxy(ProxyDispatchInput { method: &method, path: &path, settings: &state.settings, - kv: None, - ec_context: &mut ec_context, services, req, }) @@ -499,8 +497,7 @@ fn named_route_handler( Ok(admin_ec_lookup_not_supported()) } NamedRouteHandler::AdminEidsLookup => { - let partner_registry = - PartnerRegistry::from_config(&state.settings.ec.partners)?; + let partner_registry = PartnerRegistry::from_ec(&state.settings.ec)?; handle_admin_eids_lookup(&partner_registry, &req) } NamedRouteHandler::LegacyAdminDenied => Ok(legacy_admin_alias_denied()), @@ -746,14 +743,33 @@ fn trace_metadata(state: &AppState, request: &Request) -> TraceMetadata { geo: None, } } + #[cfg(test)] mod trace_dispatch_tests { use super::*; + use futures::executor::block_on; use trusted_server_core::redacted::Redacted; use trusted_server_core::settings::TrustedClientIpConfig; use trusted_server_core::trace::TraceTerminalResponse; + #[test] + fn enabled_identity_provider_fails_startup() { + let mut settings = trace_settings(false); + settings.ec.identity_providers = + vec![trusted_server_core::test_support::fixtures::enabled_identity_provider()]; + + let Err(error) = build_state_with_settings(settings) else { + panic!("should refuse to start with an enabled identity provider"); + }; + + let message = format!("{error:?}"); + assert!( + message.contains("example-provider") && message.contains("Axum"), + "should name the provider and the adapter: {message}" + ); + } + fn trace_settings(enabled: bool) -> Settings { Settings::from_toml(&format!( r#" diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 668b025ccb..44694f1baa 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -204,6 +204,7 @@ fn build_state_with_services( settings: Settings, services: Option, ) -> Result, Report> { + trusted_server_core::ec::provider::reject_enabled_providers(&settings.ec, "Cloudflare")?; let plan = Arc::new(compile_auction_plan(&settings)?); plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Cloudflare)?; let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; @@ -535,15 +536,12 @@ fn build_router(state: &Arc) -> RouterService { EdgeCacheHeader::CloudflareCdnCacheControl, ) } else if state.registry.has_route(&method, &path) { - let mut ec_context = EcContext::default(); state .registry .handle_proxy(ProxyDispatchInput { method: &method, path: &path, settings: &state.settings, - kv: None, - ec_context: &mut ec_context, services: &services, req, }) @@ -655,7 +653,7 @@ fn build_router(state: &Arc) -> RouterService { .get( "/_ts/admin/eids", make_handler(Arc::clone(&state), |s, _services, req| async move { - let partner_registry = PartnerRegistry::from_config(&s.settings.ec.partners)?; + let partner_registry = PartnerRegistry::from_ec(&s.settings.ec)?; handle_admin_eids_lookup(&partner_registry, &req) }), ) @@ -825,6 +823,23 @@ fn trace_metadata(state: &AppState, request: &Request) -> TraceMetadata { mod tests { use super::*; + #[test] + fn enabled_identity_provider_fails_startup() { + let mut settings = aps_profile_settings(); + settings.ec.identity_providers = + vec![trusted_server_core::test_support::fixtures::enabled_identity_provider()]; + + let Err(error) = build_state_with_settings(settings) else { + panic!("should refuse to start with an enabled identity provider"); + }; + + let message = format!("{error:?}"); + assert!( + message.contains("example-provider") && message.contains("Cloudflare"), + "should name the provider and the adapter: {message}" + ); + } + fn aps_profile_settings() -> Settings { let mut settings = Settings::from_toml( r#" diff --git a/crates/trusted-server-adapter-fastly/Cargo.toml b/crates/trusted-server-adapter-fastly/Cargo.toml index 6c7121d0d6..0e5d6aaceb 100644 --- a/crates/trusted-server-adapter-fastly/Cargo.toml +++ b/crates/trusted-server-adapter-fastly/Cargo.toml @@ -37,6 +37,7 @@ log-fastly = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } trusted-server-core = { workspace = true } +trusted-server-identity-providers = { workspace = true } url = { workspace = true } urlencoding = { workspace = true } web-time = { workspace = true } diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 69675e2669..bce3fcc727 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -86,6 +86,8 @@ //! [`load_settings_from_config_store`]; if settings loading itself fails, they //! are returned without geo or TS headers. +use std::cell::RefCell; +use std::net::IpAddr; use std::sync::Arc; use crate::rate_limiter::{FastlyRateLimiter, RATE_COUNTER_NAME}; @@ -97,7 +99,7 @@ use edgezero_core::http::{ HandlerFuture, HeaderValue, Method, Request, Response, StatusCode, header, }; use edgezero_core::router::RouterService; -use error_stack::Report; +use error_stack::{Report, ResultExt as _}; use trusted_server_core::auction::AuctionTelemetrySink; use trusted_server_core::auction::endpoints::handle_auction; use trusted_server_core::auction::{ @@ -111,11 +113,18 @@ use trusted_server_core::ec::admin::{ }; use trusted_server_core::ec::batch_sync::handle_batch_sync; use trusted_server_core::ec::consent::ec_consent_withdrawn; +use trusted_server_core::ec::coordinator::transport::PlatformLookupTransport; +use trusted_server_core::ec::coordinator::{ + IdentityCoordinator as _, LookupContext, PendingLookups, ProviderCoordinator, +}; use trusted_server_core::ec::device::DeviceSignals; +use trusted_server_core::ec::eid_types::UnixSeconds; use trusted_server_core::ec::identify::{cors_preflight_identify, handle_identify}; use trusted_server_core::ec::kv::KvIdentityGraph; +use trusted_server_core::ec::page_load::NavigationPermit; +use trusted_server_core::ec::provider::ClientSignals; use trusted_server_core::ec::registry::PartnerRegistry; -use trusted_server_core::ec::{EcContext, EidSyncSource}; +use trusted_server_core::ec::{EcContext, EcKvSnapshot, EidSyncSource}; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::http_util::is_navigation_request; use trusted_server_core::integrations::{ @@ -123,7 +132,8 @@ use trusted_server_core::integrations::{ RequestFilterRegistryOutcome, }; use trusted_server_core::platform::{ - ClientInfo, GeoInfo, PlatformGeo as _, PlatformKvStore, RuntimeServices, StoreName, + ClientInfo, GeoInfo, PlatformGeo as _, PlatformKvStore, PlatformPendingRequest, + RuntimeServices, StoreName, }; use trusted_server_core::proxy::{ AssetProxyCachePolicy, handle_asset_proxy_request, handle_first_party_click, @@ -142,6 +152,9 @@ use trusted_server_core::settings::{ProxyAssetRoute, Settings}; use trusted_server_core::settings_data::{DEFAULT_CONFIG_STORE_ID, get_settings_from_config_store}; use trusted_server_core::tester_cookie::{handle_clear_tester, handle_set_tester}; use trusted_server_core::trace::{TraceMetadata, TracePreDispatchHook}; +use trusted_server_identity_providers::{ + build_after_response_coordinator, build_coordinator_with, partner_registry, +}; use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware}; use crate::platform::{ @@ -190,14 +203,28 @@ pub(crate) struct AppState { pub(crate) registry: Arc, pub(crate) default_kv_store: Arc, pub(crate) auction_telemetry_sink: Arc, + /// Partner registry, built with the state. + pub(crate) partner_registry: Arc, + /// Coordinator for providers collected before the response, if any. + pub(crate) page_identity: Option>, + /// Coordinator for providers collected after the response, if any. + pub(crate) after_response_identity: Option>, +} + +impl AppState { + /// Returns the partner registry. + pub(crate) fn partner_registry(&self) -> Arc { + Arc::clone(&self.partner_registry) + } } /// Build the application state, loading settings and constructing all per-application components. /// /// # Errors /// -/// Returns an error when settings, the auction orchestrator, or the integration -/// registry fail to initialise. +/// Returns an error when settings, the auction orchestrator, the integration +/// registry, the partner registry or an enabled identity provider fail to +/// initialise. pub(crate) fn build_state( stores: &RuntimeStoreConfig, ) -> Result, Report> { @@ -229,12 +256,30 @@ pub(crate) fn build_state_from_settings( let auction_telemetry_sink = crate::tinybird::auction_sink_from_settings(&settings); let default_kv_store = Arc::new(UnavailableKvStore) as Arc; + // A provider that cannot run must stop startup: skipping it would quietly + // hand its sources back to browser ingest. + let partner_registry = Arc::new(partner_registry(&settings.ec)?); + let page_identity = build_coordinator_with(&settings, Arc::clone(&partner_registry)) + .change_context(TrustedServerError::Configuration { + message: "identity provider configuration is invalid".to_owned(), + })? + .map(Arc::new); + let after_response_identity = + build_after_response_coordinator(&settings, Arc::clone(&partner_registry)) + .change_context(TrustedServerError::Configuration { + message: "identity provider configuration is invalid".to_owned(), + })? + .map(Arc::new); + Ok(Arc::new(AppState { settings: Arc::new(settings), orchestrator: Arc::new(orchestrator), registry: Arc::new(registry), default_kv_store, auction_telemetry_sink, + partner_registry, + page_identity, + after_response_identity, })) } @@ -334,6 +379,140 @@ pub(crate) struct EcFinalizeState { pub(crate) services: RuntimeServices, } +/// Identity lookups dispatched for this navigation, collected before finalize. +pub(crate) struct IdentityLookups { + pub(crate) coordinator: Arc, + pub(crate) pending: PendingLookups, +} + +/// What a navigation's after-response identity lookups need once the response +/// has been sent. +/// +/// The router owns the request, so the request signals the providers read are +/// copied here. The lookups are dispatched only after the response, so they +/// never delay the page. +pub(crate) struct AfterResponseLookups { + pub(crate) ec_id: String, + pub(crate) client_ip: Option, + pub(crate) user_agent: Option, +} + +/// Identity work a navigation hands to the entry point. +/// +/// In-flight platform requests are not `Send`, so they cannot ride in +/// response extensions with [`EcFinalizeState`]. They travel through one +/// request-local slot instead: the entry point clears it before routing and +/// takes it after the router returns. +#[derive(Default)] +pub(crate) struct NavigationLookups { + /// Page-phase lookups, collected before EC finalization. + pub(crate) page: Option, + /// After-response lookups, dispatched once the response has been sent. + pub(crate) after_response: Option, +} + +thread_local! { + static NAVIGATION_LOOKUPS: RefCell = RefCell::new(NavigationLookups::default()); +} + +/// Takes the identity work stored for the current request. +/// +/// The entry point calls this before routing (discarding anything a prior +/// request left behind) and after the router returns. +pub(crate) fn take_navigation_lookups() -> NavigationLookups { + NAVIGATION_LOOKUPS.with(|slot| std::mem::take(&mut *slot.borrow_mut())) +} + +/// Request signals lent to identity providers. +struct ProviderRequestSignals<'a> { + client_ip: Option, + user_agent: Option<&'a str>, +} + +impl<'a> ProviderRequestSignals<'a> { + fn read(req: &'a Request, ec: &EcRequestState) -> Self { + Self { + // The trusted raw address: `EcContext::client_ip` is the + // normalized EC-hash input, which drops IPv6 separators. + client_ip: ec.services.client_info().client_ip, + user_agent: req + .headers() + .get(header::USER_AGENT) + .and_then(|value| value.to_str().ok()), + } + } +} + +/// Captures the request signals for after-response identity lookups on a +/// permitted publisher navigation. +/// +/// Returns `None` when no after-response provider is enabled, the request has +/// no EC, or the identity graph is unavailable. +fn prepare_after_response_lookups( + state: &AppState, + req: &Request, + ec: &EcRequestState, +) -> Option { + state.after_response_identity.as_ref()?; + ec.kv_graph.as_ref()?; + let ec_id = ec.ec_context.ec_value()?.to_owned(); + let signals = ProviderRequestSignals::read(req, ec); + Some(AfterResponseLookups { + ec_id, + client_ip: signals.client_ip, + user_agent: signals.user_agent.map(str::to_owned), + }) +} + +/// Starts identity-provider lookups for a permitted publisher navigation. +/// +/// Returns `None` when no provider is enabled, the request has no EC or one +/// minted on this request, or the identity graph is unavailable. +async fn dispatch_identity_lookups( + state: &AppState, + services: &RuntimeServices, + req: &Request, + ec: &mut EcRequestState, +) -> Option { + let coordinator = Arc::clone(state.page_identity.as_ref()?); + let graph = ec.kv_graph.as_ref()?; + // An EC minted on this request has no row yet and may never be issued, + // so it is not sent to vendors; a later navigation looks it up. + if ec.ec_context.ec_generated() { + return None; + } + let ec_id = ec.ec_context.ec_value()?.to_owned(); + if matches!(ec.ec_context.kv_snapshot(), EcKvSnapshot::NotRead) { + let snapshot = graph.load_snapshot(&ec_id); + ec.ec_context.set_kv_snapshot(snapshot); + } + let transport = PlatformLookupTransport::new(services); + let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); + let signals = ProviderRequestSignals::read(req, ec); + let now = trusted_server_core::ec::current_timestamp(); + let pending = coordinator + .dispatch( + &transport, + &LookupContext { + ec_id: &ec_id, + snapshot: ec.ec_context.kv_snapshot(), + consent: ec.ec_context.consent(), + client: ClientSignals { + ip: signals.client_ip, + user_agent: signals.user_agent, + }, + publisher_domain: &state.settings.publisher.domain, + rate_limiter: &limiter, + now: UnixSeconds::new(now), + }, + ) + .await; + Some(IdentityLookups { + coordinator, + pending, + }) +} + /// Per-request EC identity state built before dispatch, mirroring the /// pre-routing prelude of the legacy `route_request` (device signals, bot /// gate, cookie capture, consent/geo-aware [`EcContext`], and KV graph @@ -579,19 +758,19 @@ async fn execute_named( handler, NamedRouteHandler::AdminEcLookup | NamedRouteHandler::AdminEidsLookup ) { - let response = PartnerRegistry::from_config(&state.settings.ec.partners) - .and_then(|registry| match handler { - NamedRouteHandler::AdminEcLookup => { - // Deliberately do not use an EC request-state graph: that - // copy is bot-gated, while operators use curl for this - // authenticated diagnostic. - let kv = crate::maybe_identity_graph(&state.settings); - handle_admin_ec_lookup(kv.as_ref(), ®istry, &req) - } - NamedRouteHandler::AdminEidsLookup => handle_admin_eids_lookup(®istry, &req), - _ => unreachable!("admin diagnostics should use early dispatch"), - }) - .unwrap_or_else(|error| http_error(&error)); + let registry = state.partner_registry(); + let response = match handler { + NamedRouteHandler::AdminEcLookup => { + // Deliberately do not use an EC request-state graph: that + // copy is bot-gated, while operators use curl for this + // authenticated diagnostic. + let kv = crate::maybe_identity_graph(&state.settings); + handle_admin_ec_lookup(kv.as_ref(), ®istry, &req) + } + NamedRouteHandler::AdminEidsLookup => handle_admin_eids_lookup(®istry, &req), + _ => unreachable!("admin diagnostics should use early dispatch"), + } + .unwrap_or_else(|error| http_error(&error)); return Ok(response); } @@ -660,7 +839,7 @@ async fn run_named_route( cors_preflight_identify(&state.settings, &req) } else { let kv = crate::require_identity_graph(&state.settings)?; - let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + let partner_registry = state.partner_registry(); handle_identify( &state.settings, &kv, @@ -674,11 +853,11 @@ async fn run_named_route( NamedRouteHandler::ClearTester => handle_clear_tester(&state.settings), NamedRouteHandler::Auction => { ec.ec_context.set_eid_sync_source(EidSyncSource::Auction); - let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + let partner_registry = state.partner_registry(); let registry_ref = if partner_registry.is_empty() { None } else { - Some(&partner_registry) + Some(&*partner_registry) }; handle_auction( &state.settings, @@ -698,11 +877,11 @@ async fn run_named_route( if req.method() == Method::OPTIONS { return Ok(page_bids_preflight_denied()); } - let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + let partner_registry = state.partner_registry(); let registry_ref = if partner_registry.is_empty() { None } else { - Some(&partner_registry) + Some(&*partner_registry) }; let auction = AuctionDispatch { integration_registry: &state.registry, @@ -745,7 +924,7 @@ fn run_batch_sync(state: &AppState, services: &RuntimeServices, req: Request) -> let sharedid_cookie = crate::extract_cookie_value(&req, COOKIE_SHAREDID); let result = crate::require_identity_graph(&state.settings).and_then(|kv| { - let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + let partner_registry = state.partner_registry(); let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); handle_batch_sync(&kv, &partner_registry, &limiter, req) }); @@ -820,8 +999,6 @@ async fn dispatch_fallback( method: &method, path: &path, settings: &state.settings, - kv: ec.kv_graph.as_ref(), - ec_context: &mut ec.ec_context, services, req, }) @@ -848,11 +1025,21 @@ async fn dispatch_fallback( // Only for document navigations by recognised browsers; subresource // requests may lack consent signals such as Sec-GPC. let is_navigation = is_navigation_request(&req); + let mut has_permit = false; if is_navigation { ec.ec_context.set_eid_sync_source(EidSyncSource::Navigation); + // The page-load write permit uses the stricter Fetch Metadata test + // (spec §3.1); requests without it never write KV. + if let Some(permit) = NavigationPermit::classify(&req) { + ec.ec_context.set_navigation_permit(permit); + has_permit = true; + } } let is_publisher_navigation = ec.is_real_browser && is_navigation; + // A new EC is minted only when this request may also write its row, + // so a client without Fetch Metadata never receives a throwaway EC. if is_publisher_navigation + && has_permit && let Err(err) = ec .ec_context .generate_if_needed(&state.settings, ec.kv_graph.as_ref()) @@ -860,6 +1047,23 @@ async fn dispatch_fallback( log::warn!("EC generation failed for publisher proxy: {err:?}"); } + // Identity lookups start before the origin fetch so they overlap it; + // the entry point collects them before EC finalization. + if is_publisher_navigation + && has_permit + && let Some(lookups) = dispatch_identity_lookups(state, services, &req, &mut ec).await + { + NAVIGATION_LOOKUPS.with(|slot| slot.borrow_mut().page = Some(lookups)); + } + // After-response providers only capture request signals here; they + // are built and dispatched once the response has been sent. + if is_publisher_navigation + && has_permit + && let Some(lookups) = prepare_after_response_lookups(state, &req, &ec) + { + NAVIGATION_LOOKUPS.with(|slot| slot.borrow_mut().after_response = Some(lookups)); + } + // Run the server-side auction with the configured creative- // opportunity slots and collect dispatched bids from the lazy // publisher body stream. `handle_publisher_request` matches the @@ -867,44 +1071,40 @@ async fn dispatch_fallback( // EC identity-graph KV (`ec.kv_graph`) enrich the bid request with // server-side EIDs, same as the legacy auction. let slots = state.settings.creative_opportunity_slots(); - match PartnerRegistry::from_config(&state.settings.ec.partners) { - Ok(partner_registry) => { - let auction = AuctionDispatch { - integration_registry: &state.registry, - orchestrator: &state.orchestrator, - slots, - registry: Some(&partner_registry), - }; - match handle_publisher_request( - &state.settings, - services, - ec.kv_graph.as_ref(), - &mut ec.ec_context, - auction, - req, - EdgeCacheHeader::SurrogateControl, + let partner_registry = state.partner_registry(); + let auction = AuctionDispatch { + integration_registry: &state.registry, + orchestrator: &state.orchestrator, + slots, + registry: Some(&*partner_registry), + }; + match handle_publisher_request( + &state.settings, + services, + ec.kv_graph.as_ref(), + &mut ec.ec_context, + auction, + req, + EdgeCacheHeader::SurrogateControl, + ) + .await + { + Ok(pub_response) => { + // Origin start succeeded on the sole publisher-page + // path. Authorize orphan recovery only for real-browser + // document navigations so named routes, integration + // proxies, and filter short circuits cannot rotate an + // identity. + ec.ec_context.set_recovery_eligible(is_publisher_navigation); + publisher_response_into_streaming_response( + pub_response, + &method, + Arc::clone(&state.settings), + state.registry.as_ref(), + Arc::clone(&state.orchestrator), + services.clone(), ) .await - { - Ok(pub_response) => { - // Origin start succeeded on the sole publisher-page - // path. Authorize orphan recovery only for real-browser - // document navigations so named routes, integration - // proxies, and filter short circuits cannot rotate an - // identity. - ec.ec_context.set_recovery_eligible(is_publisher_navigation); - publisher_response_into_streaming_response( - pub_response, - &method, - Arc::clone(&state.settings), - state.registry.as_ref(), - Arc::clone(&state.orchestrator), - services.clone(), - ) - .await - } - Err(e) => Err(e), - } } Err(e) => Err(e), } @@ -1421,8 +1621,18 @@ mod tests { use error_stack::Report; use futures::StreamExt as _; use futures::executor::block_on; + use std::borrow::Cow; + use std::num::NonZeroUsize; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; + use trusted_server_core::ec::coordinator::{ProviderCoordinator, RegisteredProvider}; use trusted_server_core::ec::device::DeviceSignals; + use trusted_server_core::ec::eid_types::SourceDomain; + use trusted_server_core::ec::provider::{ + EidLevelFields, EidShape, IdentityProvider, LookupInput, LookupOutcome, + ProviderCapabilities, ProviderEid, ProviderError, ProviderExpiry, ProviderRequest, + ProviderResponse, ProviderUid, SourceCapability, SourceInput, + }; + use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::TrustedServerError; use trusted_server_core::integrations::{ HeaderMutation, IntegrationRegistry, IntegrationRequestFilter, RequestFilterDecision, @@ -1435,7 +1645,7 @@ mod tests { RuntimeServices, TemplateCacheError, TemplateCacheKey, TemplateCacheLookup, TemplateCacheMiss, TemplateCacheReservation, TemplateEntry, TemplateMetadata, }; - use trusted_server_core::settings::Settings; + use trusted_server_core::settings::{EcPartner, IdentityProviderConfig, Settings}; use trusted_server_core::test_support::nextjs_auction; #[test] @@ -1615,6 +1825,9 @@ mod tests { orchestrator: Arc::new(orchestrator), registry: Arc::new(registry), default_kv_store, + partner_registry: Arc::new(PartnerRegistry::empty()), + page_identity: None, + after_response_identity: None, }); TrustedServerApp::routes_for_state(&state) } @@ -2592,6 +2805,611 @@ mod tests { } } + fn navigation_request(path: &str) -> Request { + let mut request = browser_request(Method::GET, path, "document"); + request + .headers_mut() + .insert("sec-fetch-mode", HeaderValue::from_static("navigate")); + request + } + + fn take_finalize_state(response: &mut Response) -> super::EcFinalizeState { + response + .extensions_mut() + .remove::() + .expect("should attach finalize state") + } + + #[test] + fn publisher_navigation_receives_navigation_permit() { + let router = test_router(); + + let mut response = route(&router, navigation_request("/article")); + + let mut state = take_finalize_state(&mut response); + assert!( + state.ec_context.take_navigation_permit_for_test().is_some(), + "should issue a permit" + ); + } + + fn consented_navigation(mut request: Request) -> Request { + // Viceroy geolocates loopback to a US privacy state; a US Privacy + // string without a sale opt-out lets the navigation mint an EC. + request.extensions_mut().insert(ClientInfo { + client_ip: Some(IpAddr::V4(Ipv4Addr::LOCALHOST)), + ..ClientInfo::default() + }); + request + .headers_mut() + .insert(header::COOKIE, HeaderValue::from_static("us_privacy=1YNN")); + request + } + + #[test] + fn enabled_provider_with_unregistered_kind_fails_startup() { + let mut settings = test_settings(); + settings.ec.partners.push( + serde_json::from_value(serde_json::json!({ + "name": "Example IDs", + "source_domain": "ids.example.com" + })) + .expect("should parse partner"), + ); + settings.ec.identity_providers.push( + toml::from_str( + "id = \"example-provider\"\nkind = \"example-kind\"\nendpoint = \"https://ids.example.com/v1\"\nallowed_domains = [\"ids.example.com\"]\nsources = [\"ids.example.com\"]", + ) + .expect("should parse provider"), + ); + + let Err(error) = build_state_from_settings(settings) else { + panic!("should refuse to start with a provider this build cannot run"); + }; + + let message = format!("{error:?}"); + assert!( + message.contains("example-provider") && message.contains("example-kind"), + "should name the provider and its unregistered kind: {message}" + ); + } + + #[test] + fn navigation_without_fetch_metadata_mints_no_ec() { + let router = test_router(); + let mut request = consented_navigation(empty_request(Method::GET, "/article")); + request + .headers_mut() + .insert(header::ACCEPT, HeaderValue::from_static("text/html")); + request.extensions_mut().insert(DeviceSignals::derive( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36", + Some("t13d1516h2_8daaf6152771_b186095e22b6"), + Some("1:65536;2:0;4:6291456;6:262144"), + )); + + let mut response = route(&router, request); + + let state = take_finalize_state(&mut response); + assert!( + state.ec_context.ec_value().is_none(), + "should not mint an EC that no row can back" + ); + } + + #[test] + fn navigation_with_fetch_metadata_mints_an_ec() { + let router = test_router(); + + let mut response = route( + &router, + consented_navigation(navigation_request("/article")), + ); + + let state = take_finalize_state(&mut response); + assert!( + state.ec_context.ec_value().is_some(), + "should mint an EC on a page-load navigation" + ); + } + + #[test] + fn publisher_navigation_without_providers_dispatches_nothing() { + let router = test_router(); + + let mut response = route(&router, navigation_request("/article")); + + let _state = take_finalize_state(&mut response); + assert!( + super::take_navigation_lookups().page.is_none(), + "should dispatch no identity lookups without enabled providers" + ); + } + + /// Answers every lookup with `p-1` and counts the lookups sent. + #[derive(Default)] + struct LookupHttpClient(Arc); + + #[async_trait::async_trait(?Send)] + impl PlatformHttpClient for LookupHttpClient { + async fn send( + &self, + _request: PlatformHttpRequest, + ) -> Result> { + Err(Report::new(PlatformError::Unsupported)) + } + + async fn send_async( + &self, + request: PlatformHttpRequest, + ) -> Result> { + self.0.fetch_add(1, Ordering::SeqCst); + Ok(PlatformPendingRequest::new(()).with_backend_name(request.backend_name)) + } + + async fn select( + &self, + _pending_requests: Vec, + ) -> Result> { + Err(Report::new(PlatformError::Unsupported)) + } + + async fn select_until( + &self, + mut pending_requests: Vec, + _deadline: web_time::Instant, + ) -> Result, Report> { + let Some(ready) = pending_requests.pop() else { + return Ok(None); + }; + let backend_name = ready + .backend_name() + .expect("should carry the backend name") + .to_owned(); + let response = edgezero_core::http::response_builder() + .status(StatusCode::OK) + .body(Body::from("p-1")) + .map_err(|_| Report::new(PlatformError::HttpClient))?; + Ok(Some(PlatformSelectResult { + ready: Ok(PlatformResponse::new(response).with_backend_name(backend_name)), + remaining: pending_requests, + failed_backend_name: None, + })) + } + } + + /// Looks up `ids.example.com` and returns the response body as its UID. + struct EchoProvider(ProviderCapabilities); + + impl IdentityProvider for EchoProvider { + fn capabilities(&self) -> &ProviderCapabilities { + &self.0 + } + + fn build_request( + &self, + _lookup: &LookupInput<'_>, + ) -> Result> { + Ok(ProviderRequest { + method: Method::GET, + url: url::Url::parse("https://ids.example.com/lookup").expect("should parse"), + headers: edgezero_core::http::HeaderMap::new(), + body: Bytes::new(), + context: None, + }) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + let id = std::str::from_utf8(response.body) + .map_err(|_| Report::new(ProviderError::InvalidResponse))?; + Ok(LookupOutcome::Found(vec![ProviderEid { + source: Cow::Borrowed("ids.example.com"), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids: vec![ProviderUid { + id: Cow::Borrowed(id), + atype: None, + ext: None, + expiry: ProviderExpiry::Unspecified, + refresh: None, + }], + }])) + } + } + + struct AfterResponseFixture { + state: Arc, + sends: Arc, + graph: trusted_server_core::ec::kv::KvIdentityGraph, + } + + /// An app state whose after-response phase runs the echo provider. + fn after_response_fixture() -> AfterResponseFixture { + identity_fixture(true) + } + + /// An app state that runs the echo provider in the page or the + /// after-response phase. + fn identity_fixture(after_response: bool) -> AfterResponseFixture { + let provider_config: IdentityProviderConfig = toml::from_str( + "id = \"example-provider\"\nkind = \"echo\"\nendpoint = \"https://ids.example.com/lookup\"\nallowed_domains = [\"ids.example.com\"]\nsources = [\"ids.example.com\"]", + ) + .expect("should parse provider config"); + let partner: EcPartner = serde_json::from_value(serde_json::json!({ + "name": "ids", "source_domain": "ids.example.com", "bidstream_enabled": true + })) + .expect("should build partner"); + let mut settings = test_settings(); + settings.ec.ec_store = Some("ec_identity_store".to_owned()); + settings.ec.partners = vec![partner]; + // This build registers no kind, so the app state is built without the + // provider and the test supplies its registry and coordinator. + let partners = { + let mut with_provider = settings.ec.clone(); + with_provider.identity_providers = vec![provider_config.clone()]; + Arc::new( + PartnerRegistry::from_ec_with_kinds(&with_provider, |_| true) + .expect("should build registry"), + ) + }; + let provider: Box = Box::new(EchoProvider(ProviderCapabilities { + sources: Box::new([SourceCapability { + source: SourceDomain::try_from("ids.example.com").expect("should parse source"), + input: SourceInput::Anonymous, + }]), + shape: EidShape { + max_eids: NonZeroUsize::new(4).expect("should be non-zero"), + max_uids_per_eid: NonZeroUsize::new(4).expect("should be non-zero"), + eid_level_fields: EidLevelFields::Preserved, + }, + })); + let coordinator = ProviderCoordinator::new( + vec![RegisteredProvider::new(provider_config, provider)], + Arc::clone(&partners), + Duration::from_millis(200), + ) + .expect("should build coordinator"); + let graph = crate::require_identity_graph(&settings).expect("should open identity graph"); + let coordinator = Some(Arc::new(coordinator)); + let (page_identity, after_response_identity) = if after_response { + (None, coordinator) + } else { + (coordinator, None) + }; + let state = Arc::new(AppState { + partner_registry: partners, + page_identity, + after_response_identity, + ..Arc::into_inner(app_state_for_settings(settings)).expect("should own the app state") + }); + AfterResponseFixture { + state, + sends: Arc::default(), + graph, + } + } + + fn seed_live_row(graph: &trusted_server_core::ec::kv::KvIdentityGraph, ec_id: &str) { + let mut entry = trusted_server_core::ec::kv_types::KvEntry::tombstone( + trusted_server_core::ec::current_timestamp(), + ); + entry.consent.ok = true; + graph.create(ec_id, &entry).expect("should seed the row"); + } + + /// Routes one navigation, finalizes it and returns its after-response work. + fn finalized_navigation( + fixture: &AfterResponseFixture, + ec_id: Option<&str>, + client_ip: IpAddr, + ) -> (super::EcFinalizeState, Option) { + let (ec_state, _, lookups) = finalized_navigation_with_page(fixture, ec_id, client_ip); + (ec_state, lookups) + } + + /// Like [`finalized_navigation`], and also reports whether page-phase + /// lookups were dispatched. + fn finalized_navigation_with_page( + fixture: &AfterResponseFixture, + ec_id: Option<&str>, + client_ip: IpAddr, + ) -> ( + super::EcFinalizeState, + bool, + Option, + ) { + let router = TrustedServerApp::routes_for_state(&fixture.state); + let mut request = navigation_request("/article"); + request.extensions_mut().insert(ClientInfo { + client_ip: Some(client_ip), + ..ClientInfo::default() + }); + // Viceroy geolocates loopback to a US privacy state; a US Privacy + // string without a sale opt-out grants consent. + let cookie = ec_id.map_or_else( + || "us_privacy=1YNN".to_owned(), + |ec_id| format!("us_privacy=1YNN; ts-ec={ec_id}"), + ); + request.headers_mut().insert( + header::COOKIE, + HeaderValue::from_str(&cookie).expect("should build cookie header"), + ); + super::take_navigation_lookups(); + let mut response = route(&router, request); + let navigation = super::take_navigation_lookups(); + let lookups = navigation.after_response; + let page = navigation.page; + let page_dispatched = page.is_some(); + let mut ec_state = take_finalize_state(&mut response); + ec_state.services = RuntimeServices::builder() + .config_store(Arc::new(crate::platform::FastlyPlatformConfigStore)) + .secret_store(Arc::new(crate::platform::FastlyPlatformSecretStore)) + .kv_store(Arc::new(NoopKvStore) as Arc) + .backend(Arc::new(FixedBackend)) + .http_client(Arc::new(LookupHttpClient(Arc::clone(&fixture.sends)))) + .geo(Arc::new(crate::platform::FastlyPlatformGeo)) + .client_info(ClientInfo::default()) + .build(); + crate::apply_edgezero_ec_finalize( + &fixture.state.settings, + Some(&fixture.state), + &mut ec_state, + page, + &mut response, + ) + .expect("should finalize"); + (ec_state, page_dispatched, lookups) + } + + #[test] + fn page_phase_lookups_reach_the_write_for_a_returning_ec() { + // Arrange + let fixture = identity_fixture(false); + let ec_id = format!("{}.pg0001", "c".repeat(64)); + seed_live_row(&fixture.graph, &ec_id); + + // Act + let (_, dispatched, _) = + finalized_navigation_with_page(&fixture, Some(&ec_id), IpAddr::V4(Ipv4Addr::LOCALHOST)); + + // Assert + assert!(dispatched, "should dispatch page-phase lookups"); + assert_eq!( + owned_uid(&fixture, &ec_id).as_deref(), + Some("p-1"), + "should write the provider UID in the page-load write" + ); + } + + #[test] + fn page_phase_lookups_skip_a_newly_minted_ec() { + // Arrange + let fixture = identity_fixture(false); + + // Act + let (ec_state, dispatched, _) = + finalized_navigation_with_page(&fixture, None, IpAddr::V4(Ipv4Addr::LOCALHOST)); + + // Assert + assert!( + ec_state.ec_context.ec_value().is_some(), + "should mint an EC" + ); + assert!(!dispatched, "should not send a newly minted EC to vendors"); + } + + fn owned_uid(fixture: &AfterResponseFixture, ec_id: &str) -> Option { + let snapshot = fixture.graph.load_snapshot(ec_id); + snapshot + .entry_for(ec_id) + .and_then(|entry| entry.ids.get("ids.example.com")) + .map(|record| record.uid.clone()) + } + + #[test] + fn after_response_lookups_capture_the_raw_ipv6_address() { + // Arrange + let fixture = after_response_fixture(); + let ec_id = format!("{}.ar0006", "7".repeat(64)); + seed_live_row(&fixture.graph, &ec_id); + let ipv6 = IpAddr::V6(std::net::Ipv6Addr::new( + 0x2001, 0xdb8, 0xabcd, 1, 0, 0, 0, 0x1234, + )); + + // Act + let (_, lookups) = finalized_navigation(&fixture, Some(&ec_id), ipv6); + + // Assert + assert_eq!( + lookups.and_then(|lookups| lookups.client_ip), + Some(ipv6), + "should pass the raw IPv6 address to providers" + ); + } + + #[test] + fn after_response_lookups_write_to_a_live_row() { + // Arrange + let fixture = after_response_fixture(); + let ec_id = format!("{}.ar0001", "1".repeat(64)); + seed_live_row(&fixture.graph, &ec_id); + + // Act + let (mut ec_state, lookups) = + finalized_navigation(&fixture, Some(&ec_id), IpAddr::V4(Ipv4Addr::LOCALHOST)); + crate::run_after_response_lookups(&fixture.state, &mut ec_state, lookups); + + // Assert + assert_eq!( + fixture.sends.load(Ordering::SeqCst), + 1, + "should send one lookup" + ); + assert_eq!( + owned_uid(&fixture, &ec_id).as_deref(), + Some("p-1"), + "should write the provider UID to the row" + ); + } + + #[test] + fn after_response_lookups_skip_a_newly_minted_ec() { + // Arrange + let fixture = after_response_fixture(); + + // Act + let (mut ec_state, lookups) = + finalized_navigation(&fixture, None, IpAddr::V4(Ipv4Addr::LOCALHOST)); + let ec_id = ec_state + .ec_context + .ec_value() + .expect("should mint an EC") + .to_owned(); + crate::run_after_response_lookups(&fixture.state, &mut ec_state, lookups); + + // Assert + assert_eq!( + fixture.sends.load(Ordering::SeqCst), + 0, + "should send nothing" + ); + assert_eq!( + owned_uid(&fixture, &ec_id), + None, + "should not write provider UIDs" + ); + } + + #[test] + fn after_response_lookups_skip_when_the_ec_changed_since_capture() { + // Arrange + let fixture = after_response_fixture(); + let ec_id = format!("{}.ar0002", "2".repeat(64)); + seed_live_row(&fixture.graph, &ec_id); + let (mut ec_state, lookups) = + finalized_navigation(&fixture, Some(&ec_id), IpAddr::V4(Ipv4Addr::LOCALHOST)); + let mut lookups = lookups.expect("should capture after-response work"); + lookups.ec_id = format!("{}.ar0003", "3".repeat(64)); + // A live row for the other EC, so only the EC check can stop the send. + seed_live_row(&fixture.graph, &lookups.ec_id); + + // Act + crate::run_after_response_lookups(&fixture.state, &mut ec_state, Some(lookups)); + + // Assert + assert_eq!( + fixture.sends.load(Ordering::SeqCst), + 0, + "should send nothing" + ); + } + + #[test] + fn after_response_lookups_skip_without_a_permit() { + // Arrange + let fixture = after_response_fixture(); + let ec_id = format!("{}.ar0004", "4".repeat(64)); + seed_live_row(&fixture.graph, &ec_id); + let (mut ec_state, lookups) = + finalized_navigation(&fixture, Some(&ec_id), IpAddr::V4(Ipv4Addr::LOCALHOST)); + assert!( + ec_state.ec_context.take_after_response_permit().is_some(), + "should have issued a permit" + ); + + // Act + crate::run_after_response_lookups(&fixture.state, &mut ec_state, lookups); + + // Assert + assert_eq!( + fixture.sends.load(Ordering::SeqCst), + 0, + "should send nothing" + ); + } + + #[test] + fn after_response_lookups_skip_a_row_withdrawn_before_the_phase() { + // Arrange + let fixture = after_response_fixture(); + let ec_id = format!("{}.ar0005", "b".repeat(64)); + seed_live_row(&fixture.graph, &ec_id); + let (mut ec_state, lookups) = + finalized_navigation(&fixture, Some(&ec_id), IpAddr::V4(Ipv4Addr::LOCALHOST)); + fixture.graph.delete(&ec_id).expect("should delete the row"); + fixture + .graph + .create( + &ec_id, + &trusted_server_core::ec::kv_types::KvEntry::tombstone( + trusted_server_core::ec::current_timestamp(), + ), + ) + .expect("should write the tombstone"); + + // Act + crate::run_after_response_lookups(&fixture.state, &mut ec_state, lookups); + + // Assert + assert_eq!( + fixture.sends.load(Ordering::SeqCst), + 0, + "should send nothing" + ); + assert_eq!( + owned_uid(&fixture, &ec_id), + None, + "should keep the tombstone" + ); + } + + #[test] + fn publisher_request_without_fetch_metadata_navigation_gets_no_permit() { + let router = test_router(); + let mut prefetch = navigation_request("/article"); + prefetch + .headers_mut() + .insert("sec-purpose", HeaderValue::from_static("prefetch")); + + for request in [ + browser_request(Method::GET, "/article", "document"), + prefetch, + ] { + let mut response = route(&router, request); + + let mut state = take_finalize_state(&mut response); + assert!( + state.ec_context.take_navigation_permit_for_test().is_none(), + "should not issue a permit without a top-level navigation" + ); + } + } + + #[test] + fn integration_proxy_never_generates_ec() { + let router = test_router(); + + let mut response = route( + &router, + navigation_request("/integrations/prebid/bundle.js"), + ); + + let mut state = take_finalize_state(&mut response); + assert!( + !state.ec_context.ec_generated(), + "should not mint an EC on an integration proxy" + ); + assert!( + state.ec_context.take_navigation_permit_for_test().is_none(), + "should not issue a permit to an integration proxy" + ); + } + #[test] fn browser_device_signals_from_extension_reach_ec_finalize_state() { // Regression guard for the EdgeZero JA4/H2 signal loss: `edgezero_main` diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 3cc13fdfa8..6aafcc9c40 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -13,9 +13,19 @@ use fastly::{Request as FastlyRequest, Response as FastlyResponse}; use trusted_server_core::cache_policy::{EdgeCacheHeader, cache_control_headers_have_directive}; use trusted_server_core::constants::HEADER_X_TS_VERSION; +use trusted_server_core::ec::after_response::{ + after_response_lookups_allowed, commit_after_response_lookups, +}; +use trusted_server_core::ec::coordinator::IdentityCoordinator as _; +use trusted_server_core::ec::coordinator::LookupContext; +use trusted_server_core::ec::coordinator::transport::PlatformLookupTransport; +use trusted_server_core::ec::current_timestamp; use trusted_server_core::ec::device::DeviceSignals; +use trusted_server_core::ec::eid_types::UnixSeconds; use trusted_server_core::ec::finalize::ec_finalize_response; use trusted_server_core::ec::kv::KvIdentityGraph; +use trusted_server_core::ec::page_load::NavigationPermit; +use trusted_server_core::ec::provider::ClientSignals; use trusted_server_core::ec::pull_sync::{ PullSyncContext, build_pull_sync_context, dispatch_pull_sync, }; @@ -28,6 +38,7 @@ use trusted_server_core::response_privacy::TerminalPrivateResponse; use trusted_server_core::settings::Settings; use trusted_server_core::trace::{TraceTerminalResponse, is_trace_path}; use trusted_server_core::version_header::{apply_git_version_header, git_version_header_value}; +use trusted_server_identity_providers::partner_registry; mod app; mod backend; @@ -44,8 +55,8 @@ mod template_cache; mod tinybird; use crate::app::{ - AppState, EcFinalizeState, RuntimeStoreConfig, TrustedServerApp, - load_settings_from_config_store, + AfterResponseLookups, AppState, EcFinalizeState, IdentityLookups, NavigationLookups, + RuntimeStoreConfig, TrustedServerApp, load_settings_from_config_store, take_navigation_lookups, }; use crate::ec_kv::FastlyEcKvStore; use crate::middleware::{HEADER_X_TS_FINALIZED, apply_finalize_headers, resolve_geo_for_response}; @@ -491,6 +502,8 @@ fn edgezero_main( core_req.extensions_mut().insert(device_signals); } core_req.extensions_mut().insert(client_info); + // Drop lookups a previous request on this sandbox left behind. + drop(take_navigation_lookups()); match futures::executor::block_on(app.router().oneshot(core_req)) { Ok(response) => response, Err(error) => edge_error_response(error), @@ -517,6 +530,12 @@ fn edgezero_main( // Pop response extensions before the Fastly conversion, which drops them. let ec_state = response.extensions_mut().remove::(); + // Identity lookups dispatched on a publisher navigation are collected in + // EC finalization, after the router's executor has returned. + let NavigationLookups { + page: identity, + after_response, + } = take_navigation_lookups(); let asset_cache_policy = response.extensions_mut().remove::(); let request_filter_effects = response.extensions_mut().remove::(); @@ -544,7 +563,13 @@ fn edgezero_main( if let Some(mut ec_state) = ec_state { if let Some(settings) = settings_snapshot.as_deref() { - match apply_edgezero_ec_finalize(settings, &mut ec_state, &mut response) { + match apply_edgezero_ec_finalize( + settings, + app_state.as_deref(), + &mut ec_state, + identity, + &mut response, + ) { Ok(partner_registry) => { send_edgezero_response( response, @@ -552,6 +577,9 @@ fn edgezero_main( counters.as_ref(), ); run_edgezero_pull_sync_after_send(settings, &partner_registry, &ec_state); + if let Some(state) = app_state.as_deref() { + run_after_response_lookups(state, &mut ec_state, after_response); + } return; } Err(e) => { @@ -563,7 +591,13 @@ fn edgezero_main( } else { match load_settings_from_config_store(&runtime_stores) { Ok(settings) => { - match apply_edgezero_ec_finalize(&settings, &mut ec_state, &mut response) { + match apply_edgezero_ec_finalize( + &settings, + None, + &mut ec_state, + identity, + &mut response, + ) { Ok(partner_registry) => { send_edgezero_response( response, @@ -636,12 +670,39 @@ fn apply_entry_point_finalize_headers( apply_finalize_headers(settings, geo_info.as_ref(), response); } +/// Runs EC finalization for a routed response. +/// +/// Collects `identity` lookups first. This must run outside the router's +/// `block_on`: collection drives its own executor, and nesting executors +/// panics. fn apply_edgezero_ec_finalize( settings: &Settings, + state: Option<&AppState>, ec_state: &mut EcFinalizeState, + identity: Option, response: &mut HttpResponse, -) -> Result> { - let partner_registry = PartnerRegistry::from_config(&settings.ec.partners)?; +) -> Result, Report> { + // A redirect skips the write, so do not wait for lookups it would discard. + let identity = identity.filter(|_| { + let redirect = NavigationPermit::is_redirect(response.status()); + if redirect { + log::debug!("Identity lookups dropped: redirect response skips the write"); + } + !redirect + }); + if let Some(IdentityLookups { + coordinator, + pending, + }) = identity + { + let transport = PlatformLookupTransport::new(&ec_state.services); + let results = futures::executor::block_on(coordinator.collect(&transport, pending)); + ec_state.ec_context.set_lookup_results(results); + } + let partner_registry = match state { + Some(state) => state.partner_registry(), + None => Arc::new(partner_registry(&settings.ec)?), + }; let finalize_kv_graph = if ec_state.use_finalize_kv { maybe_identity_graph(settings) } else { @@ -686,6 +747,78 @@ fn run_edgezero_pull_sync_after_send( ); } +/// Runs after-response identity lookups and applies their follow-up write. +/// +/// Called once the response has been sent, so provider latency never delays +/// the page. Skipped unless finalize confirmed the navigation (it issues the +/// follow-up permit) and the EC is still the one the lookups were prepared +/// for. +pub(crate) fn run_after_response_lookups( + state: &AppState, + ec_state: &mut EcFinalizeState, + lookups: Option, +) { + let Some(lookups) = lookups else { + return; + }; + let Some(permit) = ec_state.ec_context.take_after_response_permit() else { + log::debug!( + "After-response lookups skipped: no permit (consent, redirect or failed page-load write)" + ); + return; + }; + if ec_state.ec_context.ec_value() != Some(lookups.ec_id.as_str()) { + log::debug!("After-response lookups skipped: EC changed during the request"); + return; + } + let Some(coordinator) = state.after_response_identity.as_ref() else { + return; + }; + let settings = &*state.settings; + let Some(graph) = maybe_identity_graph(settings) else { + return; + }; + let snapshot = graph.load_snapshot(&lookups.ec_id); + if !after_response_lookups_allowed( + &snapshot, + &lookups.ec_id, + ec_state.ec_context.ec_generated(), + ) { + log::debug!("After-response lookups skipped: new EC or no live row"); + return; + } + let transport = PlatformLookupTransport::new(&ec_state.services); + let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); + let now = current_timestamp(); + let pending = futures::executor::block_on(coordinator.dispatch( + &transport, + &LookupContext { + ec_id: &lookups.ec_id, + snapshot: &snapshot, + consent: ec_state.ec_context.consent(), + client: ClientSignals { + ip: lookups.client_ip, + user_agent: lookups.user_agent.as_deref(), + }, + publisher_domain: &settings.publisher.domain, + rate_limiter: &limiter, + now: UnixSeconds::new(now), + }, + )); + if pending.is_empty() { + return; + } + let results = futures::executor::block_on(coordinator.collect(&transport, pending)); + let outcome = commit_after_response_lookups( + &graph, + permit, + &state.partner_registry, + &lookups.ec_id, + &results, + ); + log::info!("After-response lookups: write outcome={outcome}"); +} + fn prepare_pull_sync_after_send( context: Option, graph_factory: F, diff --git a/crates/trusted-server-adapter-fastly/src/platform/deadline.rs b/crates/trusted-server-adapter-fastly/src/platform/deadline.rs index 6a8842d180..f13022c228 100644 --- a/crates/trusted-server-adapter-fastly/src/platform/deadline.rs +++ b/crates/trusted-server-adapter-fastly/src/platform/deadline.rs @@ -16,6 +16,9 @@ use super::{ // Bound payload retained across all partial bodies, as well as each response. const MAX_TOTAL_BUFFERED_BODY_BYTES: usize = MAX_PLATFORM_RESPONSE_BODY_BYTES; +/// Longest host wait an expired snapshot spends surfacing one arrived body. +const SNAPSHOT_SURFACE_TIMEOUT_MS: u32 = 1; + // Keep partially buffered responses alongside requests still awaiting headers so // an early header with a slow body cannot hold up a completed sibling response. struct DeadlinePending { @@ -215,7 +218,9 @@ fn body_is_ready(body: &BodyHandle) -> Result> { /// Dropping response bodies closes their host handles. Fastly has no pending /// request cancellation ABI: unfinished header requests are abandoned and /// never waited on past the auction deadline. If collection starts after the -/// deadline, one nonblocking snapshot preserves already available responses. +/// deadline, one snapshot preserves already available responses; it may spend +/// at most [`SNAPSHOT_SURFACE_TIMEOUT_MS`] per pending body letting the host +/// surface bytes that already arrived. /// /// Partial bodies share a 10 MiB payload budget. If a chunk exceeds that /// capacity, the largest partial response fails with an attributed HTTP client @@ -257,14 +262,26 @@ pub(super) fn select_until( } let expired_at_entry = Instant::now() >= deadline; if expired_at_entry { - // Take one nonblocking snapshot. Fully buffered results are cached; - // responses that would require a wait are abandoned, never retried. + // Take one final snapshot. Fully buffered results are cached; each + // body may wait once, at most `SNAPSHOT_SURFACE_TIMEOUT_MS`, for bytes + // that already arrived. Anything still pending is abandoned. for index in 0..pending.len() { + let mut surfaced = false; loop { let previous = pending[index].buffered_bytes(); + let was_headers = matches!(pending[index].state, Some(PendingState::Headers(_))); let result = pending[index].poll(); - let progressed = - matches!(result, Ok(None)) && pending[index].buffered_bytes() > previous; + let progressed = matches!(result, Ok(None)) + && snapshot_progressed(was_headers, previous, &pending[index]); + if matches!(result, Ok(None)) + && !progressed + && !surfaced + && let Some(PendingState::Body { body, .. }) = &pending[index].state + { + surfaced = true; + surface_arrived_body(body); + continue; + } match result { Ok(None) if progressed => {} Ok(None) => { @@ -291,7 +308,7 @@ pub(super) fn select_until( for index in 0..pending.len() { if !expired_at_entry && Instant::now() >= deadline { // A timed host wait may wake with a ready event at the cutoff. - // Finish with the same nonblocking snapshot as expired entry. + // Finish with the same final snapshot as expired entry. return select_until( pending .into_iter() @@ -337,6 +354,44 @@ pub(super) fn select_until( } } +/// Lets the host surface body bytes that have already arrived. +/// +/// A body whose bytes the host holds can still report not ready until the +/// host services async I/O once, so an expired snapshot would abandon a +/// response that fully arrived before the deadline. One select capped at +/// [`SNAPSHOT_SURFACE_TIMEOUT_MS`] runs at most once per pending body; a +/// failure leaves the body not ready. +fn surface_arrived_body(body: &BodyHandle) { + let handles = [ + // SAFETY: the live owned body retains the handle while select borrows it. + unsafe { body.as_u32() }, + ]; + let mut ready_index = 0; + // SAFETY: handles is a live slice, and ready_index is writable. The + // timeout is positive: the host interprets zero as an unbounded wait. + let status = unsafe { + fastly_sys::fastly_async_io::select( + handles.as_ptr(), + handles.len(), + SNAPSHOT_SURFACE_TIMEOUT_MS, + &mut ready_index, + ) + }; + if let Err(status) = status.result() { + log::debug!("Expired snapshot: body surface select failed: {status:?}"); + } +} + +/// Whether a nonblocking poll that returned no response advanced `item`. +/// +/// Arriving headers move a request into [`PendingState::Body`] without +/// reading any bytes; that is progress, or a response that fully arrived +/// before an expired collection would be abandoned unread. +fn snapshot_progressed(was_headers: bool, previous_bytes: usize, item: &DeadlinePending) -> bool { + let entered_body = was_headers && matches!(item.state, Some(PendingState::Body { .. })); + entered_body || item.buffered_bytes() > previous_bytes +} + fn enforce_aggregate_budget(pending: &mut [DeadlinePending], buffered_bytes: &mut usize) { if *buffered_bytes <= MAX_TOTAL_BUFFERED_BODY_BYTES { return; @@ -543,6 +598,20 @@ mod tests { ); } + #[test] + fn snapshot_counts_arriving_headers_as_progress() { + let pending = DeadlinePending::from_platform(buffered_pending(Vec::new(), "example")) + .expect("should resume a buffered pending request"); + assert!( + super::snapshot_progressed(true, 0, &pending), + "should keep polling a request whose headers just arrived" + ); + assert!( + !super::snapshot_progressed(false, 0, &pending), + "should stop polling a body that read nothing" + ); + } + #[test] fn expired_deadline_preserves_an_already_ready_body() { let pending = buffered_pending(b"ready".to_vec(), "example"); diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 9edce69fa3..dd1a20a014 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -133,6 +133,7 @@ fn build_state_with_services( settings: Settings, services: Option, ) -> Result, Report> { + trusted_server_core::ec::provider::reject_enabled_providers(&settings.ec, "Spin")?; let plan = Arc::new(compile_auction_plan(&settings)?); plan.validate_for_target(trusted_server_core::platform::AuctionTargetId::Spin)?; let orchestrator = build_orchestrator_with_plan(Arc::clone(&plan), &settings)?; @@ -653,7 +654,7 @@ fn build_router(state: &Arc) -> RouterService { let s = Arc::clone(&s); async move { let req = ctx.into_request(); - let result = PartnerRegistry::from_config(&s.settings.ec.partners) + let result = PartnerRegistry::from_ec(&s.settings.ec) .and_then(|registry| handle_admin_eids_lookup(®istry, &req)); Ok::(result.unwrap_or_else(|e| http_error(&e))) } @@ -817,15 +818,12 @@ fn build_router(state: &Arc) -> RouterService { let result = if method == Method::GET && path.starts_with("/static/tsjs=") { handle_tsjs_dynamic(&req, &state.registry, EdgeCacheHeader::SMaxageFallback) } else if state.registry.has_route(&method, &path) { - let mut ec_context = EcContext::default(); state .registry .handle_proxy(ProxyDispatchInput { method: &method, path: &path, settings: &state.settings, - kv: None, - ec_context: &mut ec_context, services: &services, req, }) @@ -1012,6 +1010,23 @@ fn trace_metadata(state: &AppState, request: &Request) -> TraceMetadata { mod tests { use super::*; + #[test] + fn enabled_identity_provider_fails_startup() { + let mut settings = multi_provider_settings(); + settings.ec.identity_providers = + vec![trusted_server_core::test_support::fixtures::enabled_identity_provider()]; + + let Err(error) = build_state_with_settings(settings) else { + panic!("should refuse to start with an enabled identity provider"); + }; + + let message = format!("{error:?}"); + assert!( + message.contains("example-provider") && message.contains("Spin"), + "should name the provider and the adapter: {message}" + ); + } + fn multi_provider_settings() -> Settings { let mut settings = Settings::from_toml( r#" diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 086515fa20..1155a786da 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -9,14 +9,15 @@ use serde_json::Value as JsonValue; use crate::auction::formats::AdRequest; use crate::auction::orchestrator::OrchestrationResult; -use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; +use crate::consent::{ConsentContext, consent_allows_server_side_auction, gate_eids_by_consent}; use crate::constants::COOKIE_TS_EIDS; use crate::cookies::extract_cookie_value; use crate::ec::EcContext; use crate::ec::EcKvSnapshot; -use crate::ec::eids::{resolve_partner_ids, to_eids}; +use crate::ec::current_timestamp; +use crate::ec::eid_types::UnixSeconds; use crate::ec::kv::KvIdentityGraph; -use crate::ec::kv_types::MAX_UID_LENGTH; +use crate::ec::kv_types::{MAX_STORED_UID_LENGTH, MAX_UID_LENGTH}; use crate::ec::prebid_eids::parse_prebid_eids_cookie; use crate::ec::registry::PartnerRegistry; use crate::error::TrustedServerError; @@ -376,6 +377,7 @@ pub async fn handle_auction( // Merge current-request client EIDs with KV-resolved EIDs, then apply // consent gating before attaching them to the auction request. + let client_eids = drop_vendor_denied_client_eids(client_eids, registry, ec_context.consent()); let merged_eids = merge_auction_eids(client_eids, eids); let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); auction_request.user.eids = @@ -535,8 +537,13 @@ pub(crate) fn resolve_auction_eids( return Some(Vec::new()); } - let resolved = resolve_partner_ids(registry, entry); - Some(to_eids(&resolved)) + let now = UnixSeconds::new(current_timestamp()); + Some( + entry + .live_eids(registry, ec_context.consent(), now) + .map(Eid::from) + .collect(), + ) } pub(crate) fn resolve_client_auction_eids( @@ -600,7 +607,7 @@ fn parse_client_auction_eids(raw: Option<&JsonValue>) -> Option> { continue; } - eids.push(Eid { source, uids }); + eids.push(Eid::new(source, uids)); } if eids.is_empty() { None } else { Some(eids) } @@ -631,12 +638,35 @@ fn parse_client_auction_uid(raw: &JsonValue) -> Option { Some(Uid { id, atype, ext }) } +/// Drops client EIDs whose source is owned by a configured identity provider +/// whose TCF vendor consent is withheld. +/// +/// Only sources marked by a configured provider carry a vendor ID, so with no +/// providers configured the input is returned unchanged. +pub(crate) fn drop_vendor_denied_client_eids( + client_eids: Option>, + registry: Option<&PartnerRegistry>, + consent: &ConsentContext, +) -> Option> { + let Some(registry) = registry else { + return client_eids; + }; + let mut eids = client_eids?; + eids.retain(|eid| { + !registry + .get(&eid.source) + .is_some_and(|partner| partner.owner_vendor_consent_denied(consent)) + }); + Some(eids) +} + pub(crate) fn merge_auction_eids( client_eids: Option>, resolved_eids: Option>, ) -> Option> { - let mut merged = Vec::new(); + let mut merged: Vec = Vec::new(); + // Server-resolved entries come first so their fields win on duplicates. for eid in resolved_eids .into_iter() .flatten() @@ -645,50 +675,67 @@ pub(crate) fn merge_auction_eids( if eid.source.is_empty() { continue; } - - let source_index = match merged - .iter() - .position(|existing: &Eid| existing.source == eid.source) - { - Some(index) => index, - None => { - merged.push(Eid { - source: eid.source.clone(), - uids: Vec::new(), - }); - merged.len() - 1 - } - }; - - for uid in eid.uids { - if uid.id.trim().is_empty() || uid.id.len() > MAX_UID_LENGTH { + let Eid { + source, + inserter, + matcher, + mm, + ext, + uids, + } = eid; + let mut kept = Vec::with_capacity(uids.len()); + for uid in uids { + // Stored provider UIDs may reach the stored limit; client UIDs + // were already capped at `MAX_UID_LENGTH` when parsed. + if uid.id.trim().is_empty() || uid.id.len() > MAX_STORED_UID_LENGTH { continue; } - - if let Some(existing_uid) = merged[source_index] - .uids + // A duplicate keeps the earlier UID's fields but fills its + // missing `atype` and `ext` from the later one. + let duplicate = merged .iter_mut() - .find(|existing| existing.id == uid.id) - { - if existing_uid.atype.is_none() { - existing_uid.atype = uid.atype; - } - if existing_uid.ext.is_none() { - existing_uid.ext = uid.ext; + .filter(|existing| existing.source == source) + .flat_map(|existing| existing.uids.iter_mut()) + .chain(kept.iter_mut()) + .find(|existing: &&mut Uid| existing.id == uid.id); + match duplicate { + Some(existing) => { + if existing.atype.is_none() { + existing.atype = uid.atype; + } + if existing.ext.is_none() { + existing.ext = uid.ext; + } } - } else { - merged[source_index].uids.push(uid); + None => kept.push(uid), } } + if kept.is_empty() { + continue; + } + // Entries are distinct when any EID-level field differs, including + // `ext` (compared by JSON value), so no entry's fields are lost. + let same_entry = merged.iter_mut().find(|existing| { + existing.source == source + && existing.inserter == inserter + && existing.matcher == matcher + && existing.mm == mm + && existing.ext == ext + }); + match same_entry { + Some(existing) => existing.uids.extend(kept), + None => merged.push(Eid { + source, + inserter, + matcher, + mm, + ext, + uids: kept, + }), + } } - merged.retain(|eid| !eid.uids.is_empty()); - - if merged.is_empty() { - None - } else { - Some(merged) - } + (!merged.is_empty()).then_some(merged) } #[cfg(test)] @@ -750,6 +797,107 @@ mod tests { .build() } + fn vendor_denied_consent() -> ConsentContext { + let mut purposes = vec![false; 24]; + purposes[0] = true; + purposes[3] = true; + ConsentContext { + jurisdiction: Jurisdiction::Gdpr, + gdpr_applies: true, + tcf: Some(crate::consent::types::TcfConsent { + version: 2, + cmp_id: 1, + cmp_version: 1, + consent_screen: 0, + consent_language: "EN".to_owned(), + vendor_list_version: 1, + tcf_policy_version: 4, + created_ds: 0, + last_updated_ds: 0, + purpose_consents: purposes, + purpose_legitimate_interests: vec![false; 24], + vendor_consents: Vec::new(), + vendor_legitimate_interests: Vec::new(), + special_feature_opt_ins: vec![false; 12], + }), + ..ConsentContext::default() + } + } + + fn client_eids_for(sources: &[&str]) -> Option> { + Some( + sources + .iter() + .map(|source| { + Eid::new( + *source, + vec![Uid { + id: format!("uid-{source}"), + atype: Some(1), + ext: None, + }], + ) + }) + .collect(), + ) + } + + fn partner_registry(sources: &[&str]) -> PartnerRegistry { + let partners: Vec = sources + .iter() + .map(|source| { + serde_json::from_value(json!({ + "name": source, "source_domain": source, "bidstream_enabled": true + })) + .expect("should build partner") + }) + .collect(); + PartnerRegistry::from_config(&partners).expect("should build registry") + } + + #[test] + fn client_eids_of_a_vendor_denied_provider_source_are_dropped() { + // Arrange + let mut registry = partner_registry(&["ids.example.com", "other.example.com"]); + assert!( + registry.mark_provider_owned("ids.example.com", std::num::NonZeroU16::new(42)), + "should mark the provider source" + ); + let client = client_eids_for(&["ids.example.com", "other.example.com"]); + + // Act + let kept = + drop_vendor_denied_client_eids(client, Some(®istry), &vendor_denied_consent()) + .expect("should keep the list"); + + // Assert + let sources: Vec<&str> = kept.iter().map(|eid| eid.source.as_str()).collect(); + assert_eq!( + sources, + vec!["other.example.com"], + "should drop only the vendor-denied provider source" + ); + } + + #[test] + fn client_eids_are_unchanged_without_providers() { + // Arrange + let registry = partner_registry(&["ids.example.com"]); + let client = client_eids_for(&["ids.example.com", "unregistered.example.com"]); + let before = serde_json::to_string(&client).expect("should serialize"); + + // Act + let kept = + drop_vendor_denied_client_eids(client, Some(®istry), &vendor_denied_consent()); + + // Assert + assert_eq!( + serde_json::to_string(&kept).expect("should serialize"), + before, + "should leave client EIDs byte-identical with no providers" + ); + } + fn make_ec_context(jurisdiction: Jurisdiction, ec_value: Option<&str>) -> EcContext { EcContext::new_for_test( ec_value.map(str::to_owned), @@ -782,10 +930,9 @@ mod tests { #[tokio::test] async fn auction_endpoint_snapshot_is_reused_by_response_finalization() { // `/auction` loads the identity-graph row to resolve server-side EIDs. - // Finalization runs afterwards on the same EC context and ingests - // `ts-eids`/`sharedId` updates. Both must be served by a single billable - // read: before the snapshot was shared, finalization saw `NotRead` and - // paid for a second lookup. + // Finalization runs afterwards on the same EC context. It must not pay + // for a second lookup, and it never writes KV: only the page load's + // navigation may write (spec D1). let settings = create_test_settings(); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, @@ -869,21 +1016,17 @@ mod tests { .get(&ec_id) .expect("should read store") .expect("row should exist"); - assert_eq!( - stored.ids.get("sharedid.org").map(|id| id.uid.as_str()), - Some("shared-cookie-id"), - "the sharedId update must still be ingested from the shared snapshot" + assert!( + stored.ids.is_empty(), + "an auction request must leave the row unchanged" ); } #[tokio::test] - async fn auction_body_eids_reach_kv_even_when_the_ts_eids_cookie_is_absent() { - // Regression test for #1184: `/auction` sends every EID it has in the - // request body, but response finalization used to ingest identity - // graph updates only from the `ts-eids` cookie — which the browser - // caps in size and may not have sent at all. `handle_auction` must - // hand its parsed body EIDs to `ec_context` so finalization ingests - // them regardless of what the cookie carried. + async fn auction_body_eids_reach_context_but_never_kv() { + // `handle_auction` hands its parsed body EIDs to `ec_context`, but body + // EIDs go to bidders on that request only: finalization of an + // `/auction` request never writes them to KV (spec §5.1, D1). let settings = create_test_settings(); let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { enabled: true, @@ -962,10 +1105,9 @@ mod tests { .get(&ec_id) .expect("should read store") .expect("row should exist"); - assert_eq!( - stored.ids.get("id5-sync.com").map(|id| id.uid.as_str()), - Some("ID5_from_body"), - "the body's EID must be ingested into KV without a ts-eids cookie" + assert!( + stored.ids.is_empty(), + "the body's EIDs must not be persisted from an auction request" ); } @@ -1701,6 +1843,118 @@ mod tests { } } + struct EidSourceCapturingProvider { + sources: Arc>>>, + } + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for EidSourceCapturingProvider { + fn provider_name(&self) -> &str { + "eid_capturing_provider" + } + + async fn request_bids( + &self, + request: &AuctionRequest, + _context: &AuctionContext<'_>, + ) -> Result> { + *self.sources.lock().expect("should lock captured sources") = Some( + request + .user + .eids + .iter() + .flatten() + .map(|eid| eid.source.clone()) + .collect(), + ); + Err(Report::new(TrustedServerError::Auction { + message: "capture only".to_string(), + })) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + _response_time_ms: u64, + ) -> Result> { + panic!("parse_response must not run when the launch fails"); + } + + fn timeout_ms(&self) -> u32 { + 100 + } + + fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { + Some("capture-backend".to_string()) + } + } + + #[tokio::test] + async fn auction_drops_vendor_denied_body_eids() { + // Arrange: a provider with vendor 42 owns one source, consent denies it. + let settings = create_test_settings(); + let mut orchestrator = AuctionOrchestrator::new(AuctionConfig { + enabled: true, + providers: AuctionConfig::legacy_provider_map(&["eid_capturing_provider"]), + timeout_ms: 2000, + mediator: None, + ..Default::default() + }); + let sources = Arc::new(std::sync::Mutex::new(None)); + orchestrator.register_provider(Arc::new(EidSourceCapturingProvider { + sources: Arc::clone(&sources), + })); + let mut registry = partner_registry(&["ids.example.com", "other.example.com"]); + assert!( + registry.mark_provider_owned("ids.example.com", std::num::NonZeroU16::new(42)), + "should mark the provider source" + ); + let graph = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.vndbdy", "a".repeat(64)); + let mut live = crate::ec::kv_types::KvEntry::tombstone(1000); + live.consent.ok = true; + graph.create(&ec_id, &live).expect("should seed live row"); + let mut ec_context = EcContext::new_for_test(Some(ec_id), vendor_denied_consent()); + let req = Request::builder() + .method("POST") + .uri("https://publisher.example.com/auction") + .body(EdgeBody::from( + serde_json::to_vec(&json!({ + "adUnits": [ + { + "code": "div-gpt-ad-1", + "mediaTypes": { "banner": { "sizes": [[300, 250]] } } + } + ], + "eids": [ + {"source": "ids.example.com", "uids": [{"id": "owned-uid", "atype": 1}]}, + {"source": "other.example.com", "uids": [{"id": "other-uid", "atype": 1}]} + ] + })) + .expect("should serialize body"), + )) + .expect("should build auction request"); + + // Act + let _ = handle_auction( + &settings, + &orchestrator, + Some(&graph), + Some(®istry), + &mut ec_context, + &noop_services(), + req, + ) + .await; + + // Assert + assert_eq!( + *sources.lock().expect("should lock captured sources"), + Some(vec!["other.example.com".to_owned()]), + "should drop the body EID of the vendor-denied provider source" + ); + } + #[tokio::test] async fn auction_strips_client_eids_when_ec_identity_denied() { // US-state opt-out via GPC: the server-side auction consent gate still @@ -2058,33 +2312,152 @@ mod tests { ); } + #[test] + fn merge_auction_eids_keeps_distinct_eid_level_entries_per_source() { + let resolved = Some(vec![Eid { + inserter: Some("a.example.com".to_owned()), + ..Eid::new( + "ids.example.com", + vec![Uid { + id: "u1".to_owned(), + atype: Some(1), + ext: None, + }], + ) + }]); + let client = Some(vec![ + Eid { + inserter: Some("b.example.com".to_owned()), + ..Eid::new( + "ids.example.com", + vec![Uid { + id: "u2".to_owned(), + atype: Some(1), + ext: None, + }], + ) + }, + Eid::new( + "ids.example.com", + vec![Uid { + id: "u1".to_owned(), + atype: Some(2), + ext: None, + }], + ), + ]); + + let merged = merge_auction_eids(client, resolved).expect("should merge"); + + assert_eq!( + merged.len(), + 2, + "should keep entries with different inserters" + ); + assert_eq!( + merged[0].uids.len(), + 1, + "should drop duplicate u1 from client" + ); + assert_eq!( + merged[0].uids[0].atype, + Some(1), + "should keep server fields for duplicate" + ); + assert_eq!( + merged[1].uids[0].id, "u2", + "should keep distinct client uid" + ); + } + + #[test] + fn merge_auction_eids_keeps_entries_with_different_ext_separate() { + let resolved = Some(vec![Eid { + ext: Some(json!({ "k": "server" })), + ..Eid::new( + "ids.example.com", + vec![Uid { + id: "u1".to_owned(), + atype: Some(1), + ext: None, + }], + ) + }]); + let client = Some(vec![Eid { + ext: Some(json!({ "k": "client" })), + ..Eid::new( + "ids.example.com", + vec![Uid { + id: "u2".to_owned(), + atype: Some(1), + ext: None, + }], + ) + }]); + + let merged = merge_auction_eids(client, resolved).expect("should merge"); + + assert_eq!(merged.len(), 2, "should keep entries with different ext"); + assert_eq!( + merged[0].ext, + Some(json!({ "k": "server" })), + "should keep server entry ext" + ); + assert_eq!( + merged[1].ext, + Some(json!({ "k": "client" })), + "should keep client entry ext" + ); + assert_eq!(merged[1].uids[0].id, "u2", "should keep client uid"); + } + + #[test] + fn merge_auction_eids_keeps_a_stored_provider_uid_past_the_ingest_limit() { + let long = "p".repeat(MAX_UID_LENGTH + 100); + let resolved_eids = Some(vec![Eid::new( + "ids.example.com".to_string(), + vec![Uid { + id: long.clone(), + atype: Some(1), + ext: None, + }], + )]); + + let merged = merge_auction_eids(None, resolved_eids).expect("should merge EIDs"); + + assert_eq!( + merged[0].uids[0].id, long, + "should send a stored UID within the stored limit" + ); + } + #[test] fn merge_auction_eids_deduplicates_client_and_resolved_ids() { - let client_eids = Some(vec![Eid { - source: "id5-sync.com".to_string(), - uids: vec![Uid { + let client_eids = Some(vec![Eid::new( + "id5-sync.com".to_string(), + vec![Uid { id: "ID5_abc".to_string(), atype: Some(1), ext: None, }], - }]); + )]); let resolved_eids = Some(vec![ - Eid { - source: "id5-sync.com".to_string(), - uids: vec![Uid { + Eid::new( + "id5-sync.com".to_string(), + vec![Uid { id: "ID5_abc".to_string(), atype: Some(1), ext: None, }], - }, - Eid { - source: "liveramp.com".to_string(), - uids: vec![Uid { + ), + Eid::new( + "liveramp.com".to_string(), + vec![Uid { id: "LR_xyz".to_string(), atype: Some(3), ext: None, }], - }, + ), ]); let merged = merge_auction_eids(client_eids, resolved_eids).expect("should merge EIDs"); @@ -2098,22 +2471,22 @@ mod tests { #[test] fn merge_auction_eids_preserves_multiple_uids_per_source() { - let client_eids = Some(vec![Eid { - source: "sharedid.org".to_string(), - uids: vec![Uid { + let client_eids = Some(vec![Eid::new( + "sharedid.org".to_string(), + vec![Uid { id: "shared_client".to_string(), atype: None, ext: None, }], - }]); - let resolved_eids = Some(vec![Eid { - source: "sharedid.org".to_string(), - uids: vec![Uid { + )]); + let resolved_eids = Some(vec![Eid::new( + "sharedid.org".to_string(), + vec![Uid { id: "shared_server".to_string(), atype: Some(3), ext: None, }], - }]); + )]); let merged = merge_auction_eids(client_eids, resolved_eids).expect("should merge EIDs"); @@ -2125,22 +2498,22 @@ mod tests { #[test] fn merge_auction_eids_prefers_server_resolved_metadata_on_conflict() { - let client_eids = Some(vec![Eid { - source: "adserver.org".to_string(), - uids: vec![Uid { + let client_eids = Some(vec![Eid::new( + "adserver.org".to_string(), + vec![Uid { id: "shared_uid".to_string(), atype: Some(1), ext: Some(json!({ "provider": "client" })), }], - }]); - let resolved_eids = Some(vec![Eid { - source: "adserver.org".to_string(), - uids: vec![Uid { + )]); + let resolved_eids = Some(vec![Eid::new( + "adserver.org".to_string(), + vec![Uid { id: "shared_uid".to_string(), atype: Some(3), ext: Some(json!({ "provider": "server" })), }], - }]); + )]); let merged = merge_auction_eids(client_eids, resolved_eids).expect("should merge EIDs"); @@ -2158,6 +2531,40 @@ mod tests { ); } + #[test] + fn merge_auction_eids_fills_missing_atype_and_ext_from_duplicate() { + let client_eids = Some(vec![Eid::new( + "adserver.org".to_string(), + vec![Uid { + id: "shared_uid".to_string(), + atype: Some(1), + ext: Some(json!({ "provider": "client" })), + }], + )]); + let resolved_eids = Some(vec![Eid::new( + "adserver.org".to_string(), + vec![Uid { + id: "shared_uid".to_string(), + atype: None, + ext: None, + }], + )]); + + let merged = merge_auction_eids(client_eids, resolved_eids).expect("should merge EIDs"); + + assert_eq!(merged[0].uids.len(), 1, "should deduplicate duplicate uid"); + assert_eq!( + merged[0].uids[0].atype, + Some(1), + "should fill missing atype from the duplicate" + ); + assert_eq!( + merged[0].uids[0].ext, + Some(json!({ "provider": "client" })), + "should fill missing ext from the duplicate" + ); + } + #[test] fn auction_rejects_oversized_body() { futures::executor::block_on(async { diff --git a/crates/trusted-server-core/src/auction/formats.rs b/crates/trusted-server-core/src/auction/formats.rs index 0c49c1e534..6ffdc7e171 100644 --- a/crates/trusted-server-core/src/auction/formats.rs +++ b/crates/trusted-server-core/src/auction/formats.rs @@ -850,14 +850,14 @@ mod tests { #[test] fn response_includes_eid_headers_when_eids_present() { let mut request = make_auction_request(); - request.user.eids = Some(vec![Eid { - source: "ssp.com".to_owned(), - uids: vec![Uid { + request.user.eids = Some(vec![Eid::new( + "ssp.com".to_owned(), + vec![Uid { id: "uid-1".to_owned(), atype: Some(3), ext: None, }], - }]); + )]); let settings = make_settings(); let result = make_empty_result(); diff --git a/crates/trusted-server-core/src/auction/test_support.rs b/crates/trusted-server-core/src/auction/test_support.rs index a83f8899a8..c839a582eb 100644 --- a/crates/trusted-server-core/src/auction/test_support.rs +++ b/crates/trusted-server-core/src/auction/test_support.rs @@ -92,14 +92,14 @@ pub(crate) fn canonical_parity_auction_request() -> AuctionRequest { raw_ac_string: Some("fictional-ac".to_string()), ..Default::default() }), - eids: Some(vec![Eid { - source: "identity.example".to_string(), - uids: vec![Uid { + eids: Some(vec![Eid::new( + "identity.example".to_string(), + vec![Uid { id: "fictional-uid".to_string(), atype: Some(1), ext: None, }], - }]), + )]), }, device: Some(DeviceInfo { user_agent: Some("Fictional Browser".to_string()), diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 50d5e9d8a9..057ebe9e20 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -14,6 +14,7 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer}; use url::{Host, Url}; use validator::{Validate, ValidationError, ValidationErrors}; +use crate::ec::provider::validate_enabled_configs; use crate::ec::registry::PartnerRegistry; use crate::error::TrustedServerError; use crate::integrations::{ @@ -33,7 +34,8 @@ use crate::integrations::{ sourcepoint::SourcepointConfig, testlight::TestlightConfig, }; -use crate::settings::{AssetOriginAuth, IntegrationConfig, IntegrationSettings, Settings}; +use crate::redacted::Redacted; +use crate::settings::{AssetOriginAuth, Ec, IntegrationConfig, IntegrationSettings, Settings}; const DEPLOY_VALIDATION_FIELD: &str = "trusted_server"; const DEPLOY_VALIDATED_INTEGRATION_IDS: &[&str] = &[ @@ -221,6 +223,16 @@ impl edgezero_core::app_config::AppConfigMeta for TrustedServerAppConfig { ], true, ), + field( + vec![ + object("ec"), + optional_object("identity_providers"), + SecretPathSegment::ArrayEach, + optional_object("secrets"), + object("token"), + ], + true, + ), field( vec![ object("handlers"), @@ -332,9 +344,30 @@ pub fn validate_settings_for_deploy(settings: &Settings) -> Result<(), Report Result<(), Report> { + match ec.identity_providers.iter().find(|config| config.enabled) { + Some(config) => Err(Report::new(TrustedServerError::Configuration { + message: format!( + "ec.identity_providers: provider `{}` is enabled, but this release registers no \ + provider kinds; set `enabled = false`", + config.id + ), + })), + None => Ok(()), + } +} + /// Runs Trusted Server runtime validation after secret references are resolved. /// /// # Errors @@ -355,7 +388,8 @@ pub fn validate_settings_for_runtime( settings.validate_admin_handler_passwords()?; let plan = crate::auction::compile_auction_plan(settings)?; validate_enabled_integrations(settings, &plan, true)?; - PartnerRegistry::from_config(&settings.ec.partners).map(|_| ())?; + PartnerRegistry::from_ec(&settings.ec).map(|_| ())?; + validate_identity_providers(&settings.ec)?; Ok(()) } @@ -387,6 +421,46 @@ fn validate_gpt_diagnostics_config(settings: &Settings) -> Result<(), Report Ec { + let mut ec = ec.clone(); + for (index, partner) in ec.partners.iter_mut().enumerate() { + if partner.api_token.is_some() { + partner.api_token = Some(Redacted::new(format!("{index:0>64}"))); + } + if partner.ts_pull_token.is_some() { + partner.ts_pull_token = Some(Redacted::new(format!("{index:1>64}"))); + } + } + ec +} + +/// Checks the identity-provider rules core owns. +/// +/// Unknown kinds and capability checks need the provider crate; the adapter +/// builds the coordinators at startup and fails to start on either. +fn validate_identity_providers(ec: &Ec) -> Result<(), Report> { + if ec.identity_providers.is_empty() { + return Ok(()); + } + let partners = PartnerRegistry::from_ec(ec)?; + validate_enabled_configs(&ec.identity_providers, &partners).map_err(|report| { + let detail = report + .frames() + .filter_map(|frame| frame.downcast_ref::()) + .map(String::as_str) + .collect::>() + .join("; "); + report.change_context(TrustedServerError::Configuration { + message: format!("ec.identity_providers: {detail}"), + }) + }) +} + fn validate_js_asset_proxy_config(settings: &Settings) -> Result<(), Report> { let Some(raw_config) = settings.integrations.get(JS_ASSET_PROXY_INTEGRATION_ID) else { return Ok(()); @@ -525,6 +599,15 @@ fn validate_secret_key_references(settings: &Settings) -> Result<(), Report Settings { + let mut settings = valid_settings(); + settings.ec.partners = vec![ + serde_json::from_value(serde_json::json!({ + "name": "ids", "source_domain": "ids.example.com" + })) + .expect("should build partner"), + ]; + let mut config = serde_json::json!({ + "id": "example-provider", + "kind": "example-kind", + "endpoint": "https://ids.example.com/v1/lookup", + "allowed_domains": ["ids.example.com"], + "sources": ["ids.example.com"], + }); + if let (Some(base), serde_json::Value::Object(overrides)) = + (config.as_object_mut(), provider) + { + base.extend(overrides); + } + settings.ec.identity_providers = + vec![serde_json::from_value(config).expect("should build provider config")]; + settings + } + + #[test] + fn runtime_validation_accepts_valid_identity_provider() { + let settings = settings_with_identity_provider(serde_json::json!({})); + + validate_settings_for_runtime(&settings).expect("should accept valid identity provider"); + } + + #[test] + fn deploy_validation_rejects_an_enabled_provider_while_no_kind_is_registered() { + let settings = settings_with_identity_provider(serde_json::json!({})); + + let err = validate_settings_for_deploy(&settings) + .expect_err("should reject an enabled provider at deploy time"); + + assert!( + err.to_string().contains("registers no provider kinds"), + "should explain why the provider is rejected: {err:?}" + ); + } + + #[test] + fn deploy_validation_accepts_provider_with_unresolved_partner_token() { + let mut settings = settings_with_identity_provider(serde_json::json!({ "enabled": false })); + settings.ec.partners[0].api_token = + Some(Redacted::new("example_partner_api_token".to_owned())); + + validate_settings_for_deploy(&settings) + .expect("should not length-check a partner token secret key name"); + } + + #[test] + fn runtime_validation_rejects_provider_endpoint_outside_allowlist() { + let settings = settings_with_identity_provider(serde_json::json!({ + "endpoint": "https://other.example.com/v1/lookup" + })); + + let err = validate_settings_for_runtime(&settings) + .expect_err("should reject endpoint outside allowlist"); + + assert!( + err.to_string().contains("allowed_domains"), + "should explain the allowlist rule: {err:?}" + ); + } + + #[test] + fn runtime_validation_rejects_provider_source_without_partner() { + let settings = settings_with_identity_provider(serde_json::json!({ + "sources": ["unknown.example.com"] + })); + + let err = validate_settings_for_runtime(&settings) + .expect_err("should reject source without partner"); + + assert!( + err.to_string().contains("[[ec.partners]]"), + "should explain the partner rule: {err:?}" + ); + } + + #[test] + fn runtime_validation_rejects_source_claimed_by_two_providers() { + let mut settings = settings_with_identity_provider(serde_json::json!({})); + let mut second = settings.ec.identity_providers[0].clone(); + second.id = "second-provider".to_owned(); + settings.ec.identity_providers.push(second); + + let err = validate_settings_for_runtime(&settings) + .expect_err("should reject double-claimed source"); + + assert!( + err.to_string() + .contains("claimed by another enabled provider"), + "should explain the claim rule: {err:?}" + ); + } + + #[test] + fn deploy_validation_rejects_empty_provider_token_reference() { + let settings = settings_with_identity_provider(serde_json::json!({ + "secrets": { "token": " " } + })); + + let err = validate_settings_for_deploy(&settings) + .expect_err("should reject empty provider token reference"); + + assert!( + err.to_string() + .contains("ec.identity_providers[0].secrets.token"), + "should identify the provider token reference: {err:?}" + ); + } + + #[test] + fn app_config_keeps_provider_token_key_reference() { + let settings = settings_with_identity_provider(serde_json::json!({ + "enabled": false, + "secrets": { "token": "example_provider_token" } + })); + + let app_config = + TrustedServerAppConfig::new(settings).expect("should validate provider token key name"); + + let serialized = serde_json::to_string(&app_config).expect("should serialize app config"); + assert!( + serialized.contains("example_provider_token"), + "should keep the provider token key reference" + ); + } + #[test] fn trace_config_requires_enabled_on_both_validation_paths() { for raw in [ diff --git a/crates/trusted-server-core/src/config_payload.rs b/crates/trusted-server-core/src/config_payload.rs index e0615a7e7a..f1084efc17 100644 --- a/crates/trusted-server-core/src/config_payload.rs +++ b/crates/trusted-server-core/src/config_payload.rs @@ -98,6 +98,28 @@ fn remove_inactive_secret_references(data: &mut serde_json::Value) { } } + if let Some(providers) = data + .pointer_mut("/ec/identity_providers") + .and_then(serde_json::Value::as_array_mut) + { + for provider in providers { + let Some(provider) = provider.as_object_mut() else { + continue; + }; + // `enabled` defaults to true, so only an explicit false disables. + let disabled = provider + .get("enabled") + .is_some_and(|enabled| !json_bool_or_string_is_true(Some(enabled))); + if disabled + && let Some(secrets) = provider + .get_mut("secrets") + .and_then(serde_json::Value::as_object_mut) + { + secrets.remove("token"); + } + } + } + let Some(datadome) = data .pointer_mut("/integrations/datadome") .and_then(serde_json::Value::as_object_mut) @@ -205,6 +227,7 @@ mod tests { "s3-session-key" => "resolved-session-token", "partner-api-token-key" => "resolved-partner-api-token-32-bytes-ok", "partner-pull-token-key" => "resolved-partner-pull-token-32-bytes-ok", + "provider-token-key" => "resolved-provider-token", "trusted-client-ip-key" => "resolved-trusted-client-ip-secret-32-bytes", "trusted-forwarder-key" => "resolved-trusted-forwarder-secret-32-bytes", _ => key, @@ -353,6 +376,46 @@ mod tests { ); } + #[test] + fn resolves_identity_provider_token_from_the_mapped_default_store() { + let mut original = test_settings(); + original.ec.partners.push( + serde_json::from_value(serde_json::json!({ + "name": "ids", "source_domain": "ids.example.com" + })) + .expect("should build partner"), + ); + original.ec.identity_providers.push( + serde_json::from_value(serde_json::json!({ + "id": "example-provider", + "kind": "example-kind", + "endpoint": "https://ids.example.com/v1/lookup", + "allowed_domains": ["ids.example.com"], + "sources": ["ids.example.com"], + "secrets": { "token": "provider-token-key" }, + })) + .expect("should build provider config"), + ); + + let reconstructed = settings_from_config_blob( + &envelope_json(&original), + &UnifiedSecretStore, + &StoreName::from("ts_secrets"), + ) + .expect("should resolve the provider token"); + + assert_eq!( + reconstructed.ec.identity_providers[0] + .secrets + .token + .as_ref() + .map(Redacted::expose) + .map(String::as_str), + Some("resolved-provider-token"), + "should resolve the provider token key" + ); + } + #[test] fn resolves_all_static_credentials_from_the_mapped_default_store() { let mut original = test_settings(); @@ -722,6 +785,12 @@ mod tests { .ec .partners .push(partner_with_pull_sync(false, "unused-partner-pull-token")); + original.ec.identity_providers.push( + toml::from_str( + "id = \"example-provider\"\nkind = \"example-kind\"\nenabled = false\nendpoint = \"https://ids.example.com/lookup\"\nallowed_domains = [\"ids.example.com\"]\nsources = [\"ids.example.com\"]\nsecrets = { token = \"unused-provider-token\" }", + ) + .expect("should parse disabled provider"), + ); let reconstructed = settings_from_config_blob( &envelope_json(&original), @@ -730,6 +799,13 @@ mod tests { ) .expect("should skip inactive optional feature references"); + assert!( + reconstructed.ec.identity_providers[0] + .secrets + .token + .is_none(), + "should not resolve a disabled provider's token" + ); assert!(reconstructed.tinybird.auction_token_secret.is_none()); assert!(reconstructed.ec.partners[0].ts_pull_token.is_none()); let datadome = reconstructed diff --git a/crates/trusted-server-core/src/consent/mod.rs b/crates/trusted-server-core/src/consent/mod.rs index 76aa6ad859..5d232df8a9 100644 --- a/crates/trusted-server-core/src/consent/mod.rs +++ b/crates/trusted-server-core/src/consent/mod.rs @@ -41,6 +41,7 @@ pub use types::{ ConsentContext, ConsentSource, PrivacyFlag, RawConsentSignals, TcfConsent, UsPrivacy, }; +use std::num::NonZeroU16; use std::sync::atomic::{AtomicBool, Ordering}; use web_time::{SystemTime, UNIX_EPOCH}; @@ -287,6 +288,23 @@ fn effective_tcf(ctx: &ConsentContext) -> Option<&types::TcfConsent> { }) } +/// Whether the effective TCF string grants vendor consent to `vendor_id`. +/// +/// Returns `false` when no TCF string applies. +#[must_use] +pub fn has_tcf_vendor_consent(ctx: &ConsentContext, vendor_id: NonZeroU16) -> bool { + effective_tcf(ctx).is_some_and(|tcf| tcf.has_vendor_consent(vendor_id.get())) +} + +/// Whether TCF rules apply and withhold vendor consent from `vendor_id`. +/// +/// Returns `false` when no TCF string applies and GDPR does not, so a TCF +/// vendor requirement only gates traffic that TCF governs. +#[must_use] +pub fn tcf_vendor_consent_denied(ctx: &ConsentContext, vendor_id: NonZeroU16) -> bool { + (ctx.gdpr_applies || effective_tcf(ctx).is_some()) && !has_tcf_vendor_consent(ctx, vendor_id) +} + /// Returns whether a server-side auction may be dispatched for this request. /// /// Fails closed for GDPR-relevant traffic: when an EU TCF signal is present @@ -685,6 +703,7 @@ mod tests { ConsentPipelineInput, allows_ec_creation, allows_eid_persistence, apply_expiration_check, apply_tcf_conflict_resolution, build_consent_context, build_context_from_signals, consent_allows_server_side_auction, gate_eids_by_consent, has_explicit_ec_withdrawal, + has_tcf_vendor_consent, }; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ @@ -692,6 +711,7 @@ mod tests { }; use crate::consent_config::{ConflictMode, ConsentConfig, ConsentMode}; use crate::cookies::parse_cookies_to_jar; + use std::num::NonZeroU16; fn build_request() -> Request { Request::builder() @@ -755,6 +775,32 @@ mod tests { } } + #[test] + fn tcf_vendor_consent_reads_the_effective_tcf_string() { + let mut tcf = TcfBuilder::new().with_storage(true).build(); + tcf.vendor_consents = vec![9999]; + let context = ConsentContext { + gdpr_applies: true, + tcf: Some(tcf), + ..ConsentContext::default() + }; + let granted = NonZeroU16::new(9999).expect("should be non-zero"); + let other = NonZeroU16::new(1234).expect("should be non-zero"); + + assert!( + has_tcf_vendor_consent(&context, granted), + "should grant consented vendor" + ); + assert!( + !has_tcf_vendor_consent(&context, other), + "should deny other vendor" + ); + assert!( + !has_tcf_vendor_consent(&ConsentContext::default(), granted), + "should deny when no TCF string applies" + ); + } + fn make_tcf(last_updated_ds: u64, allows_eids: bool) -> TcfConsent { TcfBuilder::new() .with_storage(true) diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index e21fab702c..7b8493beb2 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -26,12 +26,14 @@ use serde_json::Value as JsonValue; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt as _}; +use crate::consent::ConsentContext; use crate::constants::{COOKIE_SHAREDID, COOKIE_TS_EIDS}; use crate::cookies::extract_cookie_value; use crate::error::TrustedServerError; use crate::openrtb::Eid; -use super::eids::{resolve_partner_ids, to_eids}; +use super::current_timestamp; +use super::eid_types::UnixSeconds; use super::generation::is_valid_ec_id; use super::kv::KvIdentityGraph; use super::kv_backend::EcKvLookup; @@ -251,8 +253,8 @@ struct AuctionEidsView { struct SkippedPartnerId { /// Partner namespace key in the entry's `ids` map. source_domain: String, - /// Why the auction resolution skips it: `empty_uid`, `not_in_registry`, - /// or `bidstream_disabled`. + /// Why the auction resolution skips it: `empty_uid`, `expired` (provider + /// record with no live UID), `not_in_registry`, or `bidstream_disabled`. reason: &'static str, } @@ -394,6 +396,7 @@ fn build_lookup_response( match serde_json::from_slice::(&lookup.body) { Ok(mut entry_json) => { add_iso_timestamp_companions(&mut entry_json); + redact_refresh_state(&mut entry_json); payload.entry = Some(entry_json); match serde_json::from_slice::(&lookup.body) { @@ -470,6 +473,30 @@ fn add_iso_timestamp_companions(entry_json: &mut JsonValue) { } } +/// Placeholder served in place of a provider's private refresh state. +const REDACTED: &str = "[redacted]"; + +/// Replaces every provider refresh `state` in the entry with [`REDACTED`]. +/// +/// Refresh state can hold a provider credential (for example a refresh +/// token), so the admin view shows only that it exists and when it is due. +fn redact_refresh_state(value: &mut JsonValue) { + match value { + JsonValue::Object(object) => { + if let Some(state) = object + .get_mut("refresh") + .and_then(JsonValue::as_object_mut) + .and_then(|refresh| refresh.get_mut("state")) + { + *state = JsonValue::String(REDACTED.to_owned()); + } + object.values_mut().for_each(redact_refresh_state); + } + JsonValue::Array(items) => items.iter_mut().for_each(redact_refresh_state), + _ => {} + } +} + /// Formats a unix-seconds timestamp as ISO 8601 (`yyyy-MM-ddTHH:mm:ss.SSSZ`). /// /// Returns `None` for values outside the representable date range. @@ -479,22 +506,31 @@ fn iso_timestamp(unix_seconds: u64) -> Option { .map(|datetime| datetime.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string()) } -/// Derives the auction EID view for a valid entry, mirroring the filters in -/// [`resolve_partner_ids`] and reporting why each stored ID was skipped. +/// Derives the auction EID view for a valid entry through the same +/// [`KvEntry::live_eids`] resolution the live auction uses, reporting why each +/// stored ID was skipped. +/// +/// Vendor consent belongs to the visitor's request, which this diagnostic +/// does not have, so no source is withheld for it here. fn build_auction_view(registry: &PartnerRegistry, entry: &KvEntry) -> AuctionEidsView { - let resolved = resolve_partner_ids(registry, entry); - let eids = to_eids(&resolved); + let now = UnixSeconds::new(current_timestamp()); + let eids = entry + .live_eids(registry, &ConsentContext::default(), now) + .map(Eid::from) + .collect(); let mut skipped = Vec::new(); for (source_domain, partner_uid) in &entry.ids { - let reason = if partner_uid.uid.is_empty() { - "empty_uid" - } else { + let reason = if partner_uid.has_live_uid(now) { match registry.get(source_domain) { None => "not_in_registry", Some(partner) if !partner.bidstream_enabled => "bidstream_disabled", Some(_) => continue, } + } else if partner_uid.is_provider_written() { + "expired" + } else { + "empty_uid" }; skipped.push(SkippedPartnerId { source_domain: source_domain.clone(), @@ -682,9 +718,10 @@ mod tests { use crate::ec::kv::TombstoneOutcome; use super::*; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, UidValue}; use crate::ec::kv_backend::test_support::InMemoryEcKv; use crate::ec::kv_backend::{EcKvStore as _, EcKvWrite, EcKvWriteMode}; - use crate::ec::kv_types::KvPartnerId; + use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid}; use crate::redacted::Redacted; use crate::settings::EcPartner; @@ -778,15 +815,11 @@ mod tests { let mut entry = KvEntry::minimal("bidstream.example", "uid-live", 1_741_824_000); entry.ids.insert( "disabled.example".to_owned(), - KvPartnerId { - uid: "uid-disabled".to_owned(), - }, + KvPartnerId::single("uid-disabled".to_owned()), ); entry.ids.insert( "unknown.example".to_owned(), - KvPartnerId { - uid: "uid-unknown".to_owned(), - }, + KvPartnerId::single("uid-unknown".to_owned()), ); entry } @@ -1416,7 +1449,7 @@ mod tests { #[test] fn eids_lookup_reports_configured_source_without_valid_uid() { - let oversized_uid = "x".repeat(513); + let oversized_uid = "x".repeat(crate::ec::kv_types::MAX_STORED_UID_LENGTH + 1); let cookie = eids_cookie_for(&serde_json::json!([{ "source": "bidstream.example", "uids": [ @@ -1533,4 +1566,81 @@ mod tests { assert_eq!(matched[0]["source_domain"], "sharedid.org"); assert_eq!(matched[0]["uid"], "shared-uid-123"); } + + #[test] + fn auction_view_matches_live_auction_for_provider_records() { + let registry = PartnerRegistry::from_config(&[ + make_test_partner("expired.example", true), + make_test_partner("live.example", true), + ]) + .expect("should build registry"); + let provider_record = |id: &str, expiry: Expiry| { + let uid = KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + expiry, + ); + KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![uid])], + UnixSeconds::new(0), + ) + .expect("should build record") + }; + let mut entry = KvEntry::tombstone(1); + entry.consent.ok = true; + entry.ids.insert( + "expired.example".to_owned(), + provider_record("old", Expiry::At(UnixSeconds::new(1))), + ); + entry.ids.insert( + "live.example".to_owned(), + provider_record("new", Expiry::Never), + ); + + let view = build_auction_view(®istry, &entry); + + let view_json = serde_json::to_value(&view.eids).expect("should serialize view eids"); + let live: Vec = entry + .live_eids( + ®istry, + &ConsentContext::default(), + UnixSeconds::new(current_timestamp()), + ) + .map(Eid::from) + .collect(); + assert_eq!( + view_json, + serde_json::to_value(&live).expect("should serialize live eids"), + "should show exactly what the live auction resolves" + ); + assert_eq!(view.eids.len(), 1, "should omit the expired provider UID"); + assert_eq!(view.eids[0].uids[0].id, "new", "should keep the live UID"); + assert_eq!(view.skipped.len(), 1, "should report the expired record"); + assert_eq!(view.skipped[0].source_domain, "expired.example"); + assert_eq!(view.skipped[0].reason, "expired", "should report expiry"); + } + + #[test] + fn redacts_provider_refresh_state() { + let mut entry = serde_json::json!({ + "ids": {"token.example": {"eids": [{"source": "token.example", "uids": [ + {"id": "token", "refresh": {"state": "secret-refresh", "from": 1_000}} + ]}]}} + }); + + redact_refresh_state(&mut entry); + + let refresh = &entry["ids"]["token.example"]["eids"][0]["uids"][0]["refresh"]; + assert_eq!(refresh["state"], REDACTED, "should hide the refresh state"); + assert_eq!( + refresh["from"], 1_000, + "should keep when the refresh is due" + ); + assert_eq!( + entry["ids"]["token.example"]["eids"][0]["uids"][0]["id"], "token", + "should keep the UID" + ); + } } diff --git a/crates/trusted-server-core/src/ec/after_response.rs b/crates/trusted-server-core/src/ec/after_response.rs new file mode 100644 index 0000000000..b46d3ea489 --- /dev/null +++ b/crates/trusted-server-core/src/ec/after_response.rs @@ -0,0 +1,638 @@ +//! The follow-up write of after-response identity lookups. +//! +//! Providers configured with `collect = "after_response"` are dispatched only +//! after the navigation's response has been sent, so their latency never +//! delays the page. Their validated results reach the row in one follow-up +//! update, guarded by the row's current generation, and are served from the +//! next page load. Browser ingest never runs here: it belongs to the +//! navigation's own write. + +use super::coordinator::results::LookupResults; +use super::eid_types::UnixSeconds; +use super::kv::KvIdentityGraph; +use super::page_load::{AfterResponseWrite, RowChange}; +use super::registry::PartnerRegistry; +use super::write_plan::{WritePlanInput, plan_write}; +use super::{EcKvSnapshot, current_timestamp, log_id}; + +/// What the follow-up write did. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub enum AfterResponseOutcome { + /// The row was updated with the after-response results. + #[display("written")] + Written, + /// The results would not change the row. + #[display("unchanged")] + Unchanged, + /// The row is missing, withdrawn or unreadable, so nothing was written. + #[display("no_row")] + NoRow, + /// The row changed since it was read; the results are dropped. + #[display("conflict")] + Conflict, + /// The store rejected the write. + #[display("failed")] + Failed, +} + +/// Whether after-response lookups may be dispatched for `ec_id`. +/// +/// They run only for an EC that existed before this navigation and whose row, +/// read after the response was sent, is live and consented. An EC minted on +/// this navigation is skipped: no partner can know it yet, and its new row +/// may not be readable so soon. Its lookups run from the next navigation. +#[must_use] +pub fn after_response_lookups_allowed( + snapshot: &EcKvSnapshot, + ec_id: &str, + ec_generated: bool, +) -> bool { + !ec_generated + && snapshot + .entry_for(ec_id) + .is_some_and(|entry| entry.consent.ok) +} + +/// Applies after-response lookup results to the row of `ec_id`. +/// +/// Re-reads the row just before writing, plans an update from `lookups` +/// alone against it, and writes it once, guarded by that read's generation. +/// Reading here rather than before dispatch keeps the window in which another +/// write can conflict to the gap between this read and the write. The +/// `permit` is consumed whether or not a write is made. +/// +/// # Performance +/// +/// At most one KV read, skipped when no lookup can change the row, and at +/// most one conditional KV write. Runs after the response has been sent, so +/// it adds no page latency. +#[must_use] +pub fn commit_after_response_lookups( + graph: &KvIdentityGraph, + permit: AfterResponseWrite, + registry: &PartnerRegistry, + ec_id: &str, + lookups: &LookupResults, +) -> AfterResponseOutcome { + // Only found UIDs or cleared sources can change the row. A newly reported + // opt-out clears every source of its provider, which claims at least one; + // a remembered one clears only sources the row still stores, so a + // navigation that finds nothing left to clear needs no re-read. + let can_change = lookups + .iter() + .any(|lookup| !lookup.entries.is_empty() || !lookup.cleared.is_empty()); + if !can_change { + return AfterResponseOutcome::Unchanged; + } + let snapshot = &graph.load_snapshot(ec_id); + if !matches!(snapshot, EcKvSnapshot::Present { .. }) { + log::info!( + "After-response lookups: no row for '{}', skipping write", + log_id(ec_id) + ); + return AfterResponseOutcome::NoRow; + } + let change = plan_write(&WritePlanInput { + ec_id, + snapshot, + new_entry: None, + browser_ingest: &[], + lookups, + registry, + now: UnixSeconds::new(current_timestamp()), + }); + let Some(change) = change else { + // A withdrawn row also plans no change. + return match snapshot.entry_for(ec_id) { + Some(entry) if entry.consent.ok => AfterResponseOutcome::Unchanged, + _ => AfterResponseOutcome::NoRow, + }; + }; + let RowChange::Update { entry, expected } = change else { + log::warn!( + "After-response lookups: unexpected row change for '{}', skipping write", + log_id(ec_id) + ); + return AfterResponseOutcome::NoRow; + }; + match graph.commit_after_response(permit, ec_id, &entry, expected) { + Ok(()) => AfterResponseOutcome::Written, + Err(report) => { + log::warn!( + "After-response write failed for '{}': {}", + log_id(ec_id), + report.current_context() + ); + match report.current_context() { + super::page_load::KvWriteError::Conflict => AfterResponseOutcome::Conflict, + _ => AfterResponseOutcome::Failed, + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ec::coordinator::results::{DropCounts, LookupStatus, ProviderLookup, SkipReason}; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, SourceDomain, UidValue}; + use crate::ec::kv::TombstoneOutcome; + use crate::ec::kv_types::{KvEid, KvEntry, KvPartnerId, KvUid}; + use crate::ec::page_load::{Generation, KvWriteError}; + use crate::settings::EcPartner; + + const EC_ID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; + + fn registry() -> PartnerRegistry { + let partners: Vec = ["browser.example.com", "ids.example.com"] + .iter() + .map(|source| { + serde_json::from_value(serde_json::json!({ + "name": source, + "source_domain": source, + "bidstream_enabled": true, + })) + .expect("should build partner") + }) + .collect(); + let mut registry = PartnerRegistry::from_config(&partners).expect("should build registry"); + assert!( + registry.mark_provider_owned("ids.example.com", None), + "should mark the provider source as owned" + ); + registry + } + + fn live_entry() -> KvEntry { + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry + } + + fn found(ec_id: &str, id: &str) -> LookupResults { + let uid = KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::Never, + ); + LookupResults::new( + ec_id, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Found { + kept: 1, + dropped: DropCounts::default(), + }, + entries: vec![( + SourceDomain::try_from("ids.example.com").expect("should build source"), + vec![KvEid::new(EidFields::default(), vec![uid])], + )], + cleared: Vec::new(), + }], + ) + } + + fn stored_uid(graph: &KvIdentityGraph, source: &str) -> Option { + graph + .load_snapshot(EC_ID) + .entry_for(EC_ID) + .and_then(|entry| entry.ids.get(source).map(|record| record.uid.clone())) + } + + #[test] + fn writes_after_response_results_to_an_existing_row() { + let graph = KvIdentityGraph::in_memory("after-response"); + let mut entry = live_entry(); + entry.ids.insert( + "browser.example.com".to_owned(), + KvPartnerId::single("b-1".to_owned()), + ); + graph.create(EC_ID, &entry).expect("should seed row"); + + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &found(EC_ID, "p-1"), + ); + + assert_eq!( + outcome, + AfterResponseOutcome::Written, + "should write the row" + ); + assert_eq!( + stored_uid(&graph, "ids.example.com").as_deref(), + Some("p-1"), + "should store the provider UID" + ); + assert_eq!( + stored_uid(&graph, "browser.example.com").as_deref(), + Some("b-1"), + "should keep the browser UID written by the page load" + ); + } + + #[test] + fn writes_against_the_row_as_read_just_before_the_write() { + // Arrange + let graph = KvIdentityGraph::in_memory("after-response-reread"); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + let read_before_dispatch = graph.load_snapshot(EC_ID); + graph + .upsert_partner_id(EC_ID, "browser.example.com", "b-2") + .expect("should rewrite the row as another request would"); + assert_ne!( + graph.load_snapshot(EC_ID), + read_before_dispatch, + "should change the row after the dispatch read" + ); + + // Act + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &found(EC_ID, "p-1"), + ); + + // Assert + assert_eq!( + outcome, + AfterResponseOutcome::Written, + "should not conflict with a write made while the lookups ran" + ); + assert_eq!( + stored_uid(&graph, "browser.example.com").as_deref(), + Some("b-2"), + "should keep the other request's write" + ); + assert_eq!( + stored_uid(&graph, "ids.example.com").as_deref(), + Some("p-1"), + "should store the provider UID" + ); + } + + #[test] + fn follow_up_write_conflicts_when_the_row_changed_after_its_read() { + // Arrange + let graph = KvIdentityGraph::in_memory("after-response-conflict"); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + let read = graph.load_snapshot(EC_ID); + let expected = Generation::of(&read, EC_ID).expect("should read a generation"); + graph + .upsert_partner_id(EC_ID, "browser.example.com", "b-2") + .expect("should rewrite the row as another request would"); + + // Act + let result = graph.commit_after_response( + AfterResponseWrite::for_test(), + EC_ID, + &live_entry(), + expected, + ); + + // Assert + let report = result.expect_err("should refuse a stale generation"); + assert!( + matches!(report.current_context(), KvWriteError::Conflict), + "should report a conflict" + ); + assert_eq!( + stored_uid(&graph, "browser.example.com").as_deref(), + Some("b-2"), + "should keep the other request's write" + ); + } + + #[test] + fn follow_up_write_never_revives_a_row_withdrawn_after_its_read() { + // Arrange: scenario 23, the row is tombstoned between read and CAS. + let graph = KvIdentityGraph::in_memory("after-response-withdrawn"); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + let read = graph.load_snapshot(EC_ID); + let expected = Generation::of(&read, EC_ID).expect("should read a generation"); + assert_eq!( + graph + .write_withdrawal_tombstone(EC_ID, drop) + .expect("should withdraw the row"), + TombstoneOutcome::Written, + "should tombstone the row" + ); + + // Act + let result = graph.commit_after_response( + AfterResponseWrite::for_test(), + EC_ID, + &live_entry(), + expected, + ); + + // Assert + assert!(result.is_err(), "should refuse to overwrite the tombstone"); + assert!( + graph + .load_snapshot(EC_ID) + .entry_for(EC_ID) + .is_none_or(|entry| !entry.consent.ok), + "should leave the row withdrawn" + ); + } + + #[test] + fn replaces_a_browser_fallback_for_an_owned_source() { + let graph = KvIdentityGraph::in_memory("after-response"); + let mut entry = live_entry(); + entry.ids.insert( + "ids.example.com".to_owned(), + KvPartnerId::single("from-browser".to_owned()), + ); + graph.create(EC_ID, &entry).expect("should seed row"); + + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &found(EC_ID, "p-1"), + ); + + assert_eq!( + outcome, + AfterResponseOutcome::Written, + "should write the row" + ); + assert_eq!( + stored_uid(&graph, "ids.example.com").as_deref(), + Some("p-1"), + "should replace the browser fallback with the provider UID" + ); + } + + #[test] + fn skips_the_write_when_the_row_already_holds_the_results() { + let graph = KvIdentityGraph::in_memory("after-response"); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + let first = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &found(EC_ID, "p-1"), + ); + assert_eq!(first, AfterResponseOutcome::Written, "should write first"); + + let second = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &found(EC_ID, "p-1"), + ); + + assert_eq!( + second, + AfterResponseOutcome::Unchanged, + "should not rewrite identical results" + ); + } + + #[test] + fn writes_nothing_without_a_row() { + let graph = KvIdentityGraph::in_memory("after-response"); + + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &found(EC_ID, "p-1"), + ); + + assert_eq!( + outcome, + AfterResponseOutcome::NoRow, + "should not create a row" + ); + assert!( + graph.load_snapshot(EC_ID).entry_for(EC_ID).is_none(), + "should leave the row absent" + ); + } + + #[test] + fn writes_nothing_to_a_withdrawn_row() { + let graph = KvIdentityGraph::in_memory("after-response"); + graph + .create(EC_ID, &KvEntry::tombstone(1000)) + .expect("should seed tombstone"); + + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &found(EC_ID, "p-1"), + ); + + assert_eq!( + outcome, + AfterResponseOutcome::NoRow, + "should not revive a tombstone" + ); + assert!( + stored_uid(&graph, "ids.example.com").is_none(), + "should not store the provider UID on a tombstone" + ); + } + + #[test] + fn drops_results_made_for_another_ec() { + let graph = KvIdentityGraph::in_memory("after-response"); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + let other = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.ABC123"; + + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &found(other, "p-1"), + ); + + assert_eq!( + outcome, + AfterResponseOutcome::Unchanged, + "should discard results dispatched for another EC" + ); + assert!( + stored_uid(&graph, "ids.example.com").is_none(), + "should not store results for another EC" + ); + } + + #[test] + fn skips_the_write_without_results() { + let graph = KvIdentityGraph::in_memory("after-response"); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &LookupResults::new(EC_ID, Vec::new()), + ); + + assert_eq!( + outcome, + AfterResponseOutcome::Unchanged, + "should make no write without results" + ); + } + + #[test] + fn skipped_results_need_no_re_read() { + // Arrange: no row, so a re-read would report `NoRow`. + let graph = KvIdentityGraph::in_memory("after-response"); + let skipped = LookupResults::new( + EC_ID, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Skipped(SkipReason::NotDue), + entries: Vec::new(), + cleared: Vec::new(), + }], + ); + + // Act + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &skipped, + ); + + // Assert + assert_eq!( + outcome, + AfterResponseOutcome::Unchanged, + "should return before reading the row" + ); + } + + #[test] + fn an_opt_out_alone_still_re_reads_and_clears_the_source() { + // Arrange: the provider's UID is stored, then it reports an opt-out. + let graph = KvIdentityGraph::in_memory("after-response"); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + let seeded = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &found(EC_ID, "p-1"), + ); + assert_eq!( + seeded, + AfterResponseOutcome::Written, + "should store the provider UID first" + ); + let opted_out = opt_out(vec![ + SourceDomain::try_from("ids.example.com").expect("should build source"), + ]); + + // Act + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &opted_out, + ); + + // Assert + assert_eq!( + outcome, + AfterResponseOutcome::Written, + "should write the opt-out" + ); + assert_eq!( + stored_uid(&graph, "ids.example.com"), + None, + "should clear the opted-out provider's source" + ); + } + + fn opt_out(cleared: Vec) -> LookupResults { + LookupResults::new( + EC_ID, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Skipped(SkipReason::OptedOut), + entries: Vec::new(), + cleared, + }], + ) + } + + #[test] + fn a_remembered_opt_out_with_nothing_to_clear_skips_the_re_read() { + // Arrange: no row, so a re-read would report `NoRow`. + let graph = KvIdentityGraph::in_memory("after-response"); + + // Act + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + &opt_out(Vec::new()), + ); + + // Assert + assert_eq!( + outcome, + AfterResponseOutcome::Unchanged, + "should return before reading the row" + ); + } + + #[test] + fn lookups_run_only_for_an_existing_consented_row() { + // Arrange + let graph = KvIdentityGraph::in_memory("after_response_allowed"); + let missing = graph.load_snapshot(EC_ID); + graph.create(EC_ID, &live_entry()).expect("should seed row"); + let present = graph.load_snapshot(EC_ID); + let withdrawn_graph = KvIdentityGraph::in_memory("after_response_withdrawn"); + withdrawn_graph + .create(EC_ID, &KvEntry::tombstone(1000)) + .expect("should seed tombstone"); + let withdrawn = withdrawn_graph.load_snapshot(EC_ID); + + // Act + Assert + assert!( + after_response_lookups_allowed(&present, EC_ID, false), + "should run for a returning EC with a live row" + ); + assert!( + !after_response_lookups_allowed(&present, EC_ID, true), + "should not run for an EC minted on this navigation" + ); + assert!( + !after_response_lookups_allowed(&missing, EC_ID, false), + "should not run when the row does not exist" + ); + assert!( + !after_response_lookups_allowed(&withdrawn, EC_ID, false), + "should not run for a withdrawn row" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/batch_sync.rs b/crates/trusted-server-core/src/ec/batch_sync.rs index 4903589b29..756cc0f490 100644 --- a/crates/trusted-server-core/src/ec/batch_sync.rs +++ b/crates/trusted-server-core/src/ec/batch_sync.rs @@ -33,6 +33,8 @@ const REASON_INVALID_EC_ID: &str = "invalid_ec_id"; const REASON_INVALID_PARTNER_UID: &str = "invalid_partner_uid"; const REASON_INELIGIBLE: &str = "ineligible"; const REASON_KV_UNAVAILABLE: &str = "kv_unavailable"; +const REASON_PROVIDER_OWNED: &str = "provider_owned"; +const REASON_ROW_FULL: &str = "row_full"; /// Maximum number of mappings allowed in a single batch request. const MAX_BATCH_SIZE: usize = 1000; @@ -167,7 +169,12 @@ fn handle_batch_sync_with_writer( } // 4. Process mappings with per-item validation and rejection reasons. - let (accepted, errors) = process_mappings(writer, &partner.source_domain, &body.mappings); + let (accepted, errors) = process_mappings_for_partner( + writer, + &partner.source_domain, + registry.is_provider_owned(&partner.source_domain), + &body.mappings, + ); let rejected = errors.len(); let status = if rejected > 0 { @@ -193,6 +200,30 @@ fn content_length_exceeds_limit(req: &Request, max_body_size: usize) - .is_some_and(|content_length| content_length > max_body_size) } +/// Processes mappings for one partner, rejecting all of them when an enabled +/// identity provider owns the partner's source (spec §8.5). +fn process_mappings_for_partner( + writer: &dyn BatchSyncWriter, + partner_id: &str, + provider_owned: bool, + mappings: &[SyncMapping], +) -> (usize, Vec) { + if !provider_owned { + return process_mappings(writer, partner_id, mappings); + } + log::info!( + "Batch sync: rejecting {} mappings for provider-owned source '{partner_id}'", + mappings.len() + ); + let errors = (0..mappings.len()) + .map(|index| MappingError { + index, + reason: REASON_PROVIDER_OWNED, + }) + .collect(); + (0, errors) +} + /// Validates all mappings, then processes each normalized EC ID once. /// /// Successful and eligibility outcomes fan out to every valid input in a @@ -248,6 +279,12 @@ fn process_mappings( Ok(UpsertResult::Written | UpsertResult::Unchanged) => { accepted += group.indexes.len(); } + Ok(UpsertResult::RowFull) => { + errors.extend(group.indexes.iter().map(|&index| MappingError { + index, + reason: REASON_ROW_FULL, + })); + } Ok(UpsertResult::NotFound | UpsertResult::ConsentWithdrawn) => { errors.extend(group.indexes.iter().map(|&index| MappingError { index, @@ -544,6 +581,48 @@ mod tests { assert_eq!(errors[1].reason, REASON_INVALID_PARTNER_UID); } + #[test] + fn provider_owned_source_rejects_every_mapping() { + let writer = MockWriter::new(vec![Ok(UpsertResult::Written), Ok(UpsertResult::Written)]); + let mappings = vec![ + mapping(&format!("{}.ABC123", "a".repeat(64)), "u1", 1), + mapping(&format!("{}.ABC123", "b".repeat(64)), "u2", 1), + ]; + + let (accepted, errors) = + process_mappings_for_partner(&writer, "ids.example.com", true, &mappings); + + assert_eq!(accepted, 0, "should accept nothing"); + assert_eq!(errors.len(), 2, "should reject each mapping"); + assert!( + errors + .iter() + .all(|error| error.reason == REASON_PROVIDER_OWNED), + "should report provider_owned" + ); + } + + #[test] + fn process_mappings_reports_a_full_row_and_continues() { + let writer = MockWriter::new(vec![Ok(UpsertResult::RowFull), Ok(UpsertResult::Written)]); + + let mappings = vec![ + mapping(&format!("{}.ABC123", "a".repeat(64)), "u1", 1), + mapping(&format!("{}.ABC123", "b".repeat(64)), "u2", 1), + ]; + + let (accepted, errors) = process_mappings(&writer, "partner", &mappings); + + assert_eq!(accepted, 1, "should accept the mapping after the full row"); + assert_eq!(errors.len(), 1, "should report only the full row"); + assert_eq!(errors[0].index, 0, "should report the full row's index"); + assert_eq!( + errors[0].reason, REASON_ROW_FULL, + "should give a full row its own reason" + ); + assert_eq!(writer.calls().len(), 2, "should continue past a full row"); + } + #[test] fn process_mappings_aborts_on_kv_unavailable() { let writer = MockWriter::new(vec![ diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs new file mode 100644 index 0000000000..c0cd83359e --- /dev/null +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -0,0 +1,1724 @@ +//! Server-side identity lookup coordination (spec §8). +//! +//! [`ProviderCoordinator`] owns eligibility, consent, destination checks, a +//! deadline-bounded fan-out through a [`LookupTransport`], validation and +//! planning of the navigation's single row change. Providers stay pure. + +pub mod results; +pub mod transport; +mod validate; + +use std::collections::HashSet; +use std::num::{NonZeroU16, NonZeroU32}; +use std::sync::Arc; +use std::time::Duration; + +use error_stack::Report; + +use self::results::{LookupResults, LookupStatus, ProviderLookup, SkipReason}; +use self::transport::{ + Deadline, LookupTransport, ProviderIndex, Tagged, TransportError, TransportResponse, +}; +use self::validate::{ValidatedEntries, ValidationPolicy, validate}; +use crate::consent::ConsentContext; +use crate::consent::{allows_eid_persistence, tcf_vendor_consent_denied}; +use crate::ec::EcKvSnapshot; +use crate::ec::consent::ec_consent_granted; +use crate::ec::eid_types::{ProviderId, SourceDomain, UnixSeconds}; +use crate::ec::generation::ec_hash; +use crate::ec::kv_types::KvEntry; +use crate::ec::provider::{ + ClientSignals, IdentityProvider, LookupInput, LookupOutcome, OwnedEntries, ProviderConfigError, + ProviderEid, ProviderResponse, SourceInput, destination_allowed, parse_sources, + validate_enabled_configs, +}; +use crate::ec::rate_limiter::RateLimiter; +use crate::ec::registry::PartnerRegistry; + +use crate::settings::IdentityProviderConfig; + +/// Coordinates server-side identity lookups for one navigation. +/// +/// Implementations own eligibility, consent, destination checks, the shared +/// time budget and validation. Providers never see KV, cookies, HTTP or the +/// clock. Implementations never write KV: they return the change and the +/// caller spends the [`crate::ec::page_load::PageLoadWrite`] permit. +#[allow(async_fn_in_trait)] +pub trait IdentityCoordinator { + /// Starts lookups for every eligible provider without waiting. + /// + /// Returns empty pending lookups when no provider is enabled; skipped + /// providers are recorded and reported by [`Self::collect`]. + async fn dispatch( + &self, + transport: &T, + lookup: &LookupContext<'_>, + ) -> PendingLookups; + + /// Waits for dispatched lookups until all finish or the deadline passes, + /// then parses and validates each response. + /// + /// Never fails: per-provider failures are recorded in [`LookupResults`]. + async fn collect( + &self, + transport: &T, + pending: PendingLookups, + ) -> LookupResults; +} + +/// Request-scoped inputs to [`IdentityCoordinator::dispatch`]. +pub struct LookupContext<'a> { + /// Active EC ID. + pub ec_id: &'a str, + /// Row snapshot read before dispatch. + pub snapshot: &'a EcKvSnapshot, + /// Request consent. + pub consent: &'a ConsentContext, + /// Client signals lent to providers. + pub client: ClientSignals<'a>, + /// Publisher domain. + pub publisher_domain: &'a str, + /// Existing EC rate limiter, used only for `rate_limit_per_hour`. + pub rate_limiter: &'a dyn RateLimiter, + /// Request time. + pub now: UnixSeconds, +} + +/// Lookups in flight for one navigation. +/// +/// Only the coordinator module can create a non-empty value. `Default` +/// (empty) and `Debug` (counts only) are implemented by hand so they do not +/// require bounds on `Pending`. +pub struct PendingLookups { + requests: Vec>, + deadline: Option, + /// EC ID the lookups were dispatched for. + ec_id: String, + /// Request time, for validating `exp` and resolving relative expiry. + now: UnixSeconds, + /// Providers that finished at dispatch (skipped or failed to send). + early: Vec>, + /// Providers whose IDs were generated at dispatch, validated. + generated: Vec>, + /// Provider state to hand back with each in-flight request's response. + contexts: Vec>, +} + +impl Default for PendingLookups { + fn default() -> Self { + Self { + requests: Vec::new(), + deadline: None, + ec_id: String::new(), + now: UnixSeconds::new(0), + early: Vec::new(), + generated: Vec::new(), + contexts: Vec::new(), + } + } +} + +impl std::fmt::Debug for PendingLookups { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PendingLookups") + .field("requests", &self.requests.len()) + .field("early", &self.early.len()) + .field("generated", &self.generated.len()) + .finish_non_exhaustive() + } +} + +impl PendingLookups { + /// Whether nothing was dispatched, generated or skipped. + #[must_use] + pub fn is_empty(&self) -> bool { + self.requests.is_empty() && self.early.is_empty() && self.generated.is_empty() + } +} + +/// A configured provider handed to [`ProviderCoordinator::new`]. +pub struct RegisteredProvider

{ + /// The provider's configuration entry. + pub config: IdentityProviderConfig, + /// The built provider. + pub provider: P, +} + +impl

RegisteredProvider

{ + /// Pairs a configuration entry with its built provider. + #[must_use] + pub fn new(config: IdentityProviderConfig, provider: P) -> Self { + Self { config, provider } + } +} + +/// A provider with the operator policy the coordinator applies to it. +struct Registered

{ + id: ProviderId, + policy: ProviderPolicy, + provider: P, +} + +/// Operator policy for one provider, validated when the coordinator is built. +struct ProviderPolicy { + sources: Box<[SourceDomain]>, + allowed_domains: Box<[Box]>, + tcf_vendor_id: Option, + rate_limit_per_hour: Option, +} + +/// The production coordinator, built once at startup. +/// +/// Generic over the provider type: production uses the boxed default, tests +/// and single-provider builds use a concrete type. +pub struct ProviderCoordinator

> { + providers: Box<[Registered

]>, + /// Partner config: `openrtb_atype` for validation. + partners: Arc, + budget: Duration, +} + +impl ProviderCoordinator

{ + /// Validates and registers the enabled providers. + /// + /// Disabled providers are dropped: they make no requests. + /// + /// # Errors + /// + /// Returns [`ProviderConfigError::Invalid`] when an enabled provider's + /// endpoint is not `https` with an allowlisted host, a source has no + /// partner entry, a source is outside the provider's capabilities, a + /// source is claimed by two enabled providers, or more than `u16::MAX` + /// providers are enabled. + pub fn new( + registered: Vec>, + partners: Arc, + budget: Duration, + ) -> Result> { + let enabled: Vec> = registered + .into_iter() + .filter(|registered| registered.config.enabled) + .collect(); + if u16::try_from(enabled.len()).is_err() { + return Err(Report::new(ProviderConfigError::Invalid) + .attach("too many identity providers are enabled")); + } + validate_enabled_configs( + enabled.iter().map(|registered| ®istered.config), + &partners, + )?; + + let providers = enabled + .into_iter() + .map(Self::register) + .collect::, Report>>()?; + + Ok(Self { + providers, + partners, + budget, + }) + } + + fn register( + registered: RegisteredProvider

, + ) -> Result, Report> { + let RegisteredProvider { config, provider } = registered; + let sources = parse_sources(&config)?; + let capable: HashSet<&SourceDomain> = provider + .capabilities() + .sources + .iter() + .map(|capability| &capability.source) + .collect(); + if let Some(source) = sources.iter().find(|source| !capable.contains(source)) { + return Err(Report::new(ProviderConfigError::Invalid).attach(format!( + "source `{source}` of provider `{}` is outside its kind's capabilities", + config.id + ))); + } + Ok(Registered { + id: ProviderId::new(&config.id), + policy: ProviderPolicy { + sources, + allowed_domains: config + .allowed_domains + .iter() + .map(|domain| Box::from(domain.as_str())) + .collect(), + tcf_vendor_id: config.tcf_vendor_id, + rate_limit_per_hour: config.rate_limit_per_hour, + }, + provider, + }) + } + + /// Whether no provider is enabled. + #[must_use] + pub fn is_empty(&self) -> bool { + self.providers.is_empty() + } + + /// Whether the row holds a live provider-written UID for `source`. + fn holds_live_uid(entry: Option<&KvEntry>, source: &str, now: UnixSeconds) -> bool { + entry + .and_then(|entry| entry.ids.get(source)) + .is_some_and(|record| record.is_provider_written() && record.has_live_uid(now)) + } + + /// Claimed sources lacking a live provider-written UID (a browser + /// fallback does not count), minus sources needing input V1 + /// never has; the flag reports whether any source was removed for input. + fn due_sources( + registered: &Registered

, + entry: Option<&KvEntry>, + now: UnixSeconds, + ) -> (Vec, bool) { + let capabilities = registered.provider.capabilities(); + let mut input_blocked = false; + let mut due = Vec::new(); + for source in ®istered.policy.sources { + let record = entry + .and_then(|entry| entry.ids.get(source.as_str())) + .filter(|record| record.is_provider_written()); + // A UID whose provider asked for a refresh makes its source due; + // the stored refresh state is the lookup's input. + if record.is_some_and(|record| record.refresh_due_for(®istered.id, now)) { + due.push(source.clone()); + continue; + } + if record.is_some_and(|record| record.has_live_uid(now)) { + continue; + } + let needs_input = capabilities + .sources + .iter() + .find(|capability| capability.source == *source) + .is_some_and(|capability| capability.input != SourceInput::Anonymous); + if needs_input { + input_blocked = true; + continue; + } + due.push(source.clone()); + } + (due, input_blocked) + } + + /// Whether the provider's hourly limit is reached. Limiter errors fail + /// open, matching pull sync. + /// + /// The bucket is the EC hash (derived from the client IP), not the full + /// EC ID, so ECs behind one IP share it and a client that clears cookies + /// cannot reset it. + fn rate_limited(registered: &Registered

, lookup: &LookupContext<'_>) -> bool { + let Some(limit) = registered.policy.rate_limit_per_hour else { + return false; + }; + let key = format!("identity:{}:{}", registered.id, ec_hash(lookup.ec_id)); + match lookup.rate_limiter.exceeded(&key, limit.get()) { + Ok(exceeded) => exceeded, + Err(report) => { + log::warn!( + "Identity lookup: provider={} rate limiter failed, allowing lookup: {report:?}", + registered.id + ); + false + } + } + } + + fn lookup( + &self, + provider: ProviderIndex, + status: LookupStatus, + entries: ValidatedEntries, + ) -> ProviderLookup { + let registered = &self.providers[provider.get()]; + let cleared = if status == LookupStatus::Skipped(SkipReason::OptedOut) { + registered.policy.sources.to_vec() + } else { + Vec::new() + }; + ProviderLookup { + provider: registered.id.clone(), + status, + entries, + cleared, + } + } + + fn interpret( + &self, + provider: ProviderIndex, + response: Result>, + context: Option<&str>, + now: UnixSeconds, + ) -> ProviderLookup { + let registered = &self.providers[provider.get()]; + let response = match response { + Ok(response) => response, + Err(report) => { + let status = match report.current_context() { + TransportError::Timeout => LookupStatus::Timeout, + TransportError::Send | TransportError::BodyTooLarge => { + log::debug!( + "Identity lookup: provider={} transport failed: {report:?}", + registered.id + ); + LookupStatus::TransportFailed + } + }; + return self.lookup(provider, status, Vec::new()); + } + }; + if matches!( + response.status, + http::StatusCode::NOT_FOUND | http::StatusCode::NO_CONTENT + ) { + return self.lookup(provider, LookupStatus::NoMatch, Vec::new()); + } + if !response.status.is_success() { + return self.lookup( + provider, + LookupStatus::HttpError(response.status), + Vec::new(), + ); + } + let parsed = registered.provider.parse_response(ProviderResponse { + status: response.status, + headers: &response.headers, + body: &response.body, + context, + }); + let eids = match parsed { + Ok(LookupOutcome::Found(eids)) => eids, + Ok(LookupOutcome::NoMatch) => { + return self.lookup(provider, LookupStatus::NoMatch, Vec::new()); + } + Ok(LookupOutcome::OptedOut) => { + return self.lookup( + provider, + LookupStatus::Skipped(SkipReason::OptedOut), + Vec::new(), + ); + } + Err(report) => { + log::debug!( + "Identity lookup: provider={} response not parsed: {}", + registered.id, + report.current_context() + ); + return self.lookup(provider, LookupStatus::InvalidResponse, Vec::new()); + } + }; + self.validated(provider, eids, now) + } + + /// Validates a provider's entries and reports them as one lookup. + fn validated( + &self, + provider: ProviderIndex, + eids: Vec>, + now: UnixSeconds, + ) -> ProviderLookup { + let registered = &self.providers[provider.get()]; + let policy = ValidationPolicy { + provider: ®istered.id, + claimed: ®istered.policy.sources, + shape: registered.provider.capabilities().shape, + partners: &self.partners, + now, + }; + let (entries, dropped) = validate(eids, &policy); + let kept = entries + .iter() + .flat_map(|(_, eids)| eids) + .map(|eid| eid.uids().len()) + .sum::(); + if kept == 0 { + log::info!( + "Identity lookup: provider={} dropped every UID ({} drops)", + registered.id, + dropped.total() + ); + return self.lookup(provider, LookupStatus::NoMatch, Vec::new()); + } + let kept = u16::try_from(kept).unwrap_or(u16::MAX); + self.lookup(provider, LookupStatus::Found { kept, dropped }, entries) + } +} + +impl IdentityCoordinator for ProviderCoordinator

{ + async fn dispatch( + &self, + transport: &T, + lookup: &LookupContext<'_>, + ) -> PendingLookups { + let mut pending = PendingLookups { + ec_id: lookup.ec_id.to_owned(), + now: lookup.now, + ..PendingLookups::default() + }; + if self.providers.is_empty() { + return pending; + } + let entry = lookup.snapshot.entry_for(lookup.ec_id); + // Missing EC or EID consent, a withdrawal tombstone, an unreadable + // row (which may hide a tombstone) or no row at all skips every + // provider: a lookup discloses the EC ID before finalize could + // decline it, and an EC without a row may never be issued. + let has_row = !matches!(lookup.snapshot, EcKvSnapshot::Failed { .. }) && entry.is_some(); + let consented = ec_consent_granted(lookup.consent) + && allows_eid_persistence(lookup.consent) + && entry.is_some_and(|entry| entry.consent.ok); + let deadline = Deadline::after(self.budget); + pending.deadline = Some(deadline); + for (index, registered) in self.providers.iter().enumerate() { + let provider = ProviderIndex::new(index); + let skip = |reason| Tagged { + provider, + value: LookupStatus::Skipped(reason), + }; + let vendor_denied = registered + .policy + .tcf_vendor_id + .is_some_and(|vendor| tcf_vendor_consent_denied(lookup.consent, vendor)); + if !has_row { + pending.early.push(skip(SkipReason::NoRow)); + continue; + } + if !consented || vendor_denied { + pending.early.push(skip(SkipReason::Consent)); + continue; + } + // A remembered opt-out keeps the provider from being called until + // it lapses. It clears only sources the row still stores, so once + // they are gone the after-response path skips its re-read. + if let Some(entry) = entry.filter(|entry| entry.opted_out(®istered.id, lookup.now)) { + pending.generated.push(Tagged { + provider, + value: ProviderLookup { + provider: registered.id.clone(), + status: LookupStatus::Skipped(SkipReason::OptedOut), + entries: Vec::new(), + cleared: registered + .policy + .sources + .iter() + .filter(|source| entry.ids.contains_key(source.as_str())) + .cloned() + .collect(), + }, + }); + continue; + } + let (due, input_blocked) = Self::due_sources(registered, entry, lookup.now); + if due.is_empty() { + pending.early.push(skip(if input_blocked { + SkipReason::Input + } else { + SkipReason::NotDue + })); + continue; + } + let input = LookupInput { + ec_id: lookup.ec_id, + consent: lookup.consent, + client: lookup.client, + publisher_domain: lookup.publisher_domain, + due_sources: &due, + current: OwnedEntries::new(entry, ®istered.policy.sources, ®istered.id), + hashed_email: None, + }; + if let Some(mut eids) = registered.provider.generate(&input) { + // Only a due source without a live UID may take a generated + // ID: a refresh-due source keeps its UID and goes to lookup. + eids.retain(|eid| { + SourceDomain::try_from(eid.source.as_ref()).is_ok_and(|source| { + due.contains(&source) + && !Self::holds_live_uid(entry, source.as_str(), lookup.now) + }) + }); + if !eids.is_empty() { + pending.generated.push(Tagged { + provider, + value: self.validated(provider, eids, lookup.now), + }); + continue; + } + } + if Self::rate_limited(registered, lookup) { + pending.early.push(skip(SkipReason::RateLimited)); + continue; + } + let mut request = match registered.provider.build_request(&input) { + Ok(request) => request, + Err(report) => { + log::warn!( + "Identity lookup: provider={} request not built: {}", + registered.id, + report.current_context() + ); + pending.early.push(Tagged { + provider, + value: LookupStatus::RequestFailed, + }); + continue; + } + }; + if !destination_allowed(&request.url, ®istered.policy.allowed_domains) { + pending.early.push(skip(SkipReason::Destination)); + continue; + } + let context = request.context.take(); + match transport.start(request, deadline).await { + Ok(value) => { + pending.requests.push(Tagged { provider, value }); + if let Some(value) = context { + pending.contexts.push(Tagged { provider, value }); + } + } + Err(report) => { + log::warn!( + "Identity lookup: provider={} send failed: {}", + registered.id, + report.current_context() + ); + pending.early.push(Tagged { + provider, + value: LookupStatus::TransportFailed, + }); + } + } + } + pending + } + + async fn collect( + &self, + transport: &T, + pending: PendingLookups, + ) -> LookupResults { + let PendingLookups { + requests, + deadline, + ec_id, + now, + early, + generated, + contexts, + } = pending; + let responses = match deadline { + Some(deadline) if !requests.is_empty() => transport.gather(requests, deadline).await, + _ => Vec::new(), + }; + let mut slots: Vec> = + (0..self.providers.len()).map(|_| None).collect(); + for Tagged { provider, value } in early { + slots[provider.get()] = Some(self.lookup(provider, value, Vec::new())); + } + for Tagged { provider, value } in generated { + slots[provider.get()] = Some(value); + } + for Tagged { provider, value } in responses { + let context = contexts + .iter() + .find(|tagged| tagged.provider == provider) + .map(|tagged| tagged.value.as_str()); + slots[provider.get()] = Some(self.interpret(provider, value, context, now)); + } + let lookups: Vec = slots.into_iter().flatten().collect(); + for lookup in &lookups { + log_outcome(lookup); + } + LookupResults::new(&ec_id, lookups) + } +} + +fn log_outcome(lookup: &ProviderLookup) { + let (kept, dropped) = match lookup.status { + LookupStatus::Found { kept, dropped } => (kept, dropped.total()), + _ => (0, 0), + }; + log::info!( + "Identity lookup: provider={} outcome={} kept={kept} dropped={dropped}", + lookup.provider, + lookup.status.metric_name() + ); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::consent::jurisdiction::Jurisdiction; + use crate::consent::types::TcfConsent; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, UidValue}; + use crate::ec::kv::KvIdentityGraph; + use crate::ec::kv_types::{KvEid, KvPartnerId, KvRefresh, KvUid}; + use crate::ec::provider::{ + EidLevelFields, EidShape, ProviderCapabilities, ProviderEid, ProviderError, ProviderExpiry, + ProviderRequest, ProviderUid, SourceCapability, + }; + use crate::error::TrustedServerError; + use crate::settings::EcPartner; + use std::borrow::Cow; + use std::cell::RefCell; + use std::num::NonZeroUsize; + + const EC_ID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; + + type Reply = fn() -> Result>; + + /// Context the echo provider sets on its request and expects back. + const ECHO_CONTEXT: &str = "echo-context"; + + struct EchoProvider(ProviderCapabilities); + + impl IdentityProvider for EchoProvider { + fn capabilities(&self) -> &ProviderCapabilities { + &self.0 + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + Ok(ProviderRequest { + method: http::Method::GET, + url: url::Url::parse(&format!( + "https://ids.example.com/lookup?due={}", + lookup.due_sources.len() + )) + .expect("should parse"), + headers: http::HeaderMap::new(), + body: bytes::Bytes::new(), + context: Some(ECHO_CONTEXT.to_owned()), + }) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + if response.context != Some(ECHO_CONTEXT) { + return Err(Report::new(ProviderError::InvalidResponse)); + } + let id = std::str::from_utf8(response.body) + .map_err(|_| Report::new(ProviderError::InvalidResponse))?; + if id == "optout" { + return Ok(LookupOutcome::OptedOut); + } + Ok(LookupOutcome::Found(vec![ProviderEid { + source: Cow::Borrowed("ids.example.com"), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids: vec![ProviderUid { + id: Cow::Borrowed(id), + atype: None, + ext: None, + expiry: ProviderExpiry::Unspecified, + refresh: None, + }], + }])) + } + } + + /// Answers every request with a canned result, recording the URLs sent. + struct CannedTransport { + reply: Reply, + sent: RefCell>, + } + + impl CannedTransport { + fn new(reply: Reply) -> Self { + Self { + reply, + sent: RefCell::default(), + } + } + } + + impl LookupTransport for CannedTransport { + type Pending = (); + + async fn start( + &self, + request: ProviderRequest, + _deadline: Deadline, + ) -> Result<(), Report> { + self.sent.borrow_mut().push(request.url.to_string()); + Ok(()) + } + + async fn gather( + &self, + pending: Vec>, + _deadline: Deadline, + ) -> Vec>>> { + pending + .into_iter() + .map(|tagged| Tagged { + provider: tagged.provider, + value: (self.reply)(), + }) + .collect() + } + } + + struct Limiter(bool); + + impl RateLimiter for Limiter { + fn exceeded( + &self, + _key: &str, + _hourly_limit: u32, + ) -> Result> { + Ok(self.0) + } + } + + fn reply(status: http::StatusCode, body: &'static [u8]) -> TransportResponse { + TransportResponse { + status, + headers: http::HeaderMap::new(), + body: bytes::Bytes::from_static(body), + } + } + + fn ok_reply() -> Result> { + Ok(reply(http::StatusCode::OK, b"p-1")) + } + + fn timeout_reply() -> Result> { + Err(Report::new(TransportError::Timeout)) + } + + fn not_found_reply() -> Result> { + Ok(reply(http::StatusCode::NOT_FOUND, b"")) + } + + fn server_error_reply() -> Result> { + Ok(reply(http::StatusCode::BAD_GATEWAY, b"")) + } + + fn invalid_reply() -> Result> { + Ok(reply(http::StatusCode::OK, b"\xff")) + } + + fn partners() -> Arc { + let partner: EcPartner = serde_json::from_value(serde_json::json!({ + "name": "ids", "source_domain": "ids.example.com", "bidstream_enabled": true + })) + .expect("should build partner"); + Arc::new(PartnerRegistry::from_config(&[partner]).expect("should build registry")) + } + + fn provider_config(extra: &str) -> IdentityProviderConfig { + toml::from_str(&format!( + "id = \"example-provider\"\nkind = \"echo\"\nendpoint = \"https://ids.example.com/lookup\"\nallowed_domains = [\"ids.example.com\"]\nsources = [\"ids.example.com\"]\n{extra}" + )) + .expect("should parse config") + } + + fn echo_provider(input: SourceInput) -> EchoProvider { + EchoProvider(ProviderCapabilities { + sources: Box::new([SourceCapability { + source: SourceDomain::try_from("ids.example.com").expect("should parse"), + input, + }]), + shape: EidShape { + max_eids: NonZeroUsize::new(4).expect("should be non-zero"), + max_uids_per_eid: NonZeroUsize::new(4).expect("should be non-zero"), + eid_level_fields: EidLevelFields::Preserved, + }, + }) + } + + fn coordinator_with( + registered: RegisteredProvider, + ) -> ProviderCoordinator { + ProviderCoordinator::new(vec![registered], partners(), Duration::from_millis(200)) + .expect("should build coordinator") + } + + fn coordinator(extra: &str) -> ProviderCoordinator { + coordinator_with(RegisteredProvider::new( + provider_config(extra), + echo_provider(SourceInput::Anonymous), + )) + } + + fn granting() -> ConsentContext { + ConsentContext { + jurisdiction: Jurisdiction::NonRegulated, + ..Default::default() + } + } + + fn missing() -> EcKvSnapshot { + EcKvSnapshot::Missing { + ec_id: EC_ID.to_owned(), + } + } + + /// A consented row with no IDs yet. + fn live_row() -> EcKvSnapshot { + let mut entry = crate::ec::kv_types::KvEntry::tombstone(1000); + entry.consent.ok = true; + EcKvSnapshot::Present { + ec_id: EC_ID.to_owned(), + entry: Box::new(entry), + generation: Some(1), + } + } + + fn run_with( + coordinator: &ProviderCoordinator

, + transport: &CannedTransport, + snapshot: &EcKvSnapshot, + consent: &ConsentContext, + limiter: &dyn RateLimiter, + ) -> LookupResults { + let context = LookupContext { + ec_id: EC_ID, + snapshot, + consent, + client: ClientSignals { + ip: None, + user_agent: None, + }, + publisher_domain: "www.example.com", + rate_limiter: limiter, + now: UnixSeconds::new(1000), + }; + futures::executor::block_on(async { + let pending = coordinator.dispatch(transport, &context).await; + coordinator.collect(transport, pending).await + }) + } + + fn run( + coordinator: &ProviderCoordinator

, + transport: &CannedTransport, + snapshot: &EcKvSnapshot, + consent: &ConsentContext, + ) -> LookupResults { + run_with(coordinator, transport, snapshot, consent, &Limiter(false)) + } + + fn only_status(results: &LookupResults) -> LookupStatus { + results + .iter() + .next() + .expect("should report the provider") + .status + } + + fn seeded_snapshot(entry: &KvEntry) -> EcKvSnapshot { + let graph = KvIdentityGraph::in_memory("coord"); + graph.create(EC_ID, entry).expect("should seed"); + graph.load_snapshot(EC_ID) + } + + #[test] + fn due_source_is_looked_up_and_validated() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &live_row(), &granting()); + + let lookup = results.iter().next().expect("should report the provider"); + assert!( + matches!(lookup.status, LookupStatus::Found { kept: 1, .. }), + "should keep one UID" + ); + assert_eq!( + lookup.entries[0].1[0].uids()[0] + .owner() + .map(ProviderId::as_str), + Some("example-provider"), + "should record the owner" + ); + assert_eq!(transport.sent.borrow().len(), 1, "should send one request"); + } + + #[test] + fn ec_without_a_row_is_not_sent_to_vendors() { + // Arrange: a well-formed `ts-ec` cookie whose row does not exist. + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + + // Act + let results = run(&coordinator, &transport, &missing(), &granting()); + + // Assert + assert_eq!( + results + .iter() + .next() + .expect("should report the provider") + .status, + LookupStatus::Skipped(SkipReason::NoRow), + "should skip a provider for an EC with no row" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + fn provider_record(id: &str) -> KvPartnerId { + let uid = KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::Never, + ); + KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![uid])], + UnixSeconds::new(1000), + ) + .expect("should build provider record") + } + + #[test] + fn source_with_browser_fallback_uid_is_still_due() { + // Arrange + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry.ids.insert( + "ids.example.com".to_owned(), + KvPartnerId::single("fallback"), + ); + + // Act + let results = run( + &coordinator, + &transport, + &seeded_snapshot(&entry), + &granting(), + ); + + // Assert + assert!( + matches!(only_status(&results), LookupStatus::Found { .. }), + "should look up a source holding only a browser fallback" + ); + assert_eq!(transport.sent.borrow().len(), 1, "should send one request"); + } + + #[test] + fn source_with_live_uid_is_not_due() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry + .ids + .insert("ids.example.com".to_owned(), provider_record("p-1")); + + let results = run( + &coordinator, + &transport, + &seeded_snapshot(&entry), + &granting(), + ); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::NotDue), + "should skip" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn tombstoned_row_skips_for_consent() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + + let results = run( + &coordinator, + &transport, + &seeded_snapshot(&KvEntry::tombstone(1000)), + &granting(), + ); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::Consent), + "should skip a tombstoned row" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn missing_ec_consent_skips_every_provider() { + let coordinator = coordinator(""); + let cases = [ + ( + "GPC opt-out in a US state", + ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + ..Default::default() + }, + ), + ( + "unknown jurisdiction", + ConsentContext { + jurisdiction: Jurisdiction::Unknown, + ..Default::default() + }, + ), + ( + "GDPR without a TC string", + ConsentContext { + jurisdiction: Jurisdiction::Gdpr, + ..Default::default() + }, + ), + ]; + + for (label, consent) in cases { + let transport = CannedTransport::new(ok_reply); + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + + let results = run(&coordinator, &transport, &seeded_snapshot(&entry), &consent); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::Consent), + "should skip for {label}" + ); + assert!( + transport.sent.borrow().is_empty(), + "should send nothing for {label}" + ); + } + } + + #[test] + fn failed_snapshot_skips_every_provider() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + let snapshot = EcKvSnapshot::Failed { + ec_id: EC_ID.to_owned(), + }; + + let results = run(&coordinator, &transport, &snapshot, &granting()); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::NoRow), + "should skip when the row could not be read" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn tcf_vendor_id_does_not_gate_traffic_without_tcf() { + let coordinator = coordinator("tcf_vendor_id = 9999"); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &live_row(), &granting()); + + assert!( + !matches!(only_status(&results), LookupStatus::Skipped(_)), + "should look up when no TCF string applies outside GDPR" + ); + assert_eq!(transport.sent.borrow().len(), 1, "should send one request"); + } + + fn tcf_consent(vendor_consents: Vec) -> ConsentContext { + let mut purposes = vec![false; 24]; + purposes[0] = true; + purposes[3] = true; + ConsentContext { + jurisdiction: Jurisdiction::Gdpr, + gdpr_applies: true, + tcf: Some(TcfConsent { + version: 2, + cmp_id: 1, + cmp_version: 1, + consent_screen: 0, + consent_language: "EN".to_owned(), + vendor_list_version: 1, + tcf_policy_version: 4, + created_ds: 0, + last_updated_ds: 0, + purpose_consents: purposes, + purpose_legitimate_interests: vec![false; 24], + vendor_consents, + vendor_legitimate_interests: Vec::new(), + special_feature_opt_ins: vec![false; 12], + }), + ..ConsentContext::default() + } + } + + #[test] + fn missing_tcf_vendor_consent_skips_before_building_a_request() { + let coordinator = coordinator("tcf_vendor_id = 9999"); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &live_row(), &tcf_consent(vec![1])); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::Consent), + "should skip" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn granted_tcf_vendor_consent_allows_the_lookup() { + let coordinator = coordinator("tcf_vendor_id = 9999"); + let transport = CannedTransport::new(ok_reply); + + let results = run( + &coordinator, + &transport, + &live_row(), + &tcf_consent(vec![9999]), + ); + + assert!( + matches!(only_status(&results), LookupStatus::Found { .. }), + "should look up with vendor consent" + ); + } + + #[test] + fn hashed_email_sources_are_skipped_for_input() { + let coordinator = coordinator_with(RegisteredProvider::new( + provider_config(""), + echo_provider(SourceInput::HashedEmail), + )); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &live_row(), &granting()); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::Input), + "should skip sources needing hashed email" + ); + } + + #[test] + fn rate_limited_provider_is_skipped() { + let coordinator = coordinator_with(RegisteredProvider::new( + provider_config("rate_limit_per_hour = 1"), + echo_provider(SourceInput::Anonymous), + )); + let transport = CannedTransport::new(ok_reply); + + let results = run_with( + &coordinator, + &transport, + &live_row(), + &granting(), + &Limiter(true), + ); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::RateLimited), + "should skip a rate-limited provider" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn built_url_outside_allowlist_is_rejected() { + let coordinator = coordinator(""); + // The echo provider builds its URL on the configured host; narrow the + // policy afterwards so the built URL falls outside the allowlist. + let coordinator = ProviderCoordinator { + providers: coordinator + .providers + .into_vec() + .into_iter() + .map(|mut registered| { + registered.policy.allowed_domains = Box::new([Box::from("other.example.com")]); + registered + }) + .collect(), + ..coordinator + }; + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &live_row(), &granting()); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::Destination), + "should reject the destination" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn transport_and_http_outcomes_map_to_statuses() { + let cases: [(Reply, LookupStatus); 4] = [ + (timeout_reply, LookupStatus::Timeout), + (not_found_reply, LookupStatus::NoMatch), + ( + server_error_reply, + LookupStatus::HttpError(http::StatusCode::BAD_GATEWAY), + ), + (invalid_reply, LookupStatus::InvalidResponse), + ]; + for (reply, expected) in cases { + let coordinator = coordinator(""); + let transport = CannedTransport::new(reply); + + let results = run(&coordinator, &transport, &live_row(), &granting()); + + let lookup = results.iter().next().expect("should report"); + assert_eq!(lookup.status, expected, "should map the outcome"); + assert!(lookup.entries.is_empty(), "should carry no entries"); + } + } + + #[test] + fn disabled_provider_is_not_registered() { + let coordinator = coordinator("enabled = false"); + + assert!(coordinator.is_empty(), "should drop disabled providers"); + } + + #[test] + fn endpoint_outside_allowlist_is_rejected_at_build() { + let mut config = provider_config(""); + config.endpoint = "https://other.example.com/".to_owned(); + + let error = ProviderCoordinator::new( + vec![RegisteredProvider::new( + config, + echo_provider(SourceInput::Anonymous), + )], + partners(), + Duration::from_millis(200), + ) + .err() + .expect("should reject endpoint"); + + assert!( + matches!(error.current_context(), ProviderConfigError::Invalid), + "should report invalid config" + ); + } + + #[test] + fn source_outside_capabilities_is_rejected_at_build() { + let provider = EchoProvider(ProviderCapabilities { + sources: Box::new([]), + ..echo_provider(SourceInput::Anonymous).0 + }); + + let error = ProviderCoordinator::new( + vec![RegisteredProvider::new(provider_config(""), provider)], + partners(), + Duration::from_millis(200), + ) + .err() + .expect("should reject source outside capabilities"); + + assert!( + matches!(error.current_context(), ProviderConfigError::Invalid), + "should report invalid config" + ); + } + + fn refreshable_record(from: u64) -> KvPartnerId { + refreshable_record_owned_by("example-provider", from) + } + + fn refreshable_record_owned_by(owner: &str, from: u64) -> KvPartnerId { + let uid = KvUid::new( + UidValue::try_from("p-1").expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new(owner)), + Expiry::At(UnixSeconds::new(5000)), + ) + .with_refresh(KvRefresh { + state: "refresh-state".to_owned(), + from: UnixSeconds::new(from), + }); + KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![uid])], + UnixSeconds::new(1000), + ) + .expect("should build provider record") + } + + fn snapshot_holding(record: KvPartnerId) -> EcKvSnapshot { + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry.ids.insert("ids.example.com".to_owned(), record); + seeded_snapshot(&entry) + } + + #[test] + fn a_due_refresh_makes_a_live_source_due() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + + let results = run( + &coordinator, + &transport, + &snapshot_holding(refreshable_record(900)), + &granting(), + ); + + assert!( + matches!(only_status(&results), LookupStatus::Found { .. }), + "should refresh a live UID whose refresh time has passed" + ); + assert_eq!(transport.sent.borrow().len(), 1, "should send the refresh"); + } + + #[test] + fn a_future_refresh_leaves_the_source_held() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + + let results = run( + &coordinator, + &transport, + &snapshot_holding(refreshable_record(2000)), + &granting(), + ); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::NotDue), + "should not refresh before the refresh time" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn remembered_opt_out_skips_the_provider_until_it_lapses() { + // Arrange + let coordinator = coordinator(""); + let provider = ProviderId::new("example-provider"); + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry.opt_outs.insert(provider.clone(), 1001); + let transport = CannedTransport::new(ok_reply); + + // Act + let results = run( + &coordinator, + &transport, + &seeded_snapshot(&entry), + &granting(), + ); + + // Assert + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::OptedOut), + "should skip a provider whose opt-out is remembered" + ); + assert!( + transport.sent.borrow().is_empty(), + "should not call a provider the visitor opted out of" + ); + assert!( + results + .iter() + .next() + .expect("should report the provider") + .cleared + .is_empty(), + "should clear nothing when the row stores none of its sources" + ); + + // Arrange: a UID for a claimed source is still stored. + let mut stored = entry.clone(); + stored + .ids + .insert("ids.example.com".to_owned(), KvPartnerId::single("p-1")); + + // Act + let results = run( + &coordinator, + &transport, + &seeded_snapshot(&stored), + &granting(), + ); + + // Assert + assert_eq!( + results + .iter() + .next() + .expect("should report the provider") + .cleared, + [SourceDomain::try_from("ids.example.com").expect("should parse")], + "should clear the claimed source the row still stores" + ); + + // Arrange: the opt-out lapsed. + entry.opt_outs.insert(provider, 1000); + + // Act + let results = run( + &coordinator, + &transport, + &seeded_snapshot(&entry), + &granting(), + ); + + // Assert + assert!( + matches!(only_status(&results), LookupStatus::Found { .. }), + "should call the provider again once the opt-out lapses" + ); + } + + #[test] + fn another_providers_refresh_state_is_never_handed_over() { + // Arrange: a live UID with due refresh state written by a provider that + // no longer owns the source. + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + + // Act + let results = run( + &coordinator, + &transport, + &snapshot_holding(refreshable_record_owned_by("former-provider", 900)), + &granting(), + ); + + // Assert + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::NotDue), + "should not refresh a UID another provider wrote" + ); + assert!( + transport.sent.borrow().is_empty(), + "should not send another provider's refresh state" + ); + } + + fn optout_reply() -> Result> { + Ok(reply(http::StatusCode::OK, b"optout")) + } + + #[test] + fn an_opted_out_reply_clears_the_sources() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(optout_reply); + + let results = run(&coordinator, &transport, &live_row(), &granting()); + + let lookup = results.iter().next().expect("should report the provider"); + assert_eq!( + lookup.status, + LookupStatus::Skipped(SkipReason::OptedOut), + "should report the opt-out" + ); + assert_eq!( + lookup.cleared, + [SourceDomain::try_from("ids.example.com").expect("should parse")], + "should clear the provider's sources" + ); + } + + /// An echo provider whose request can never be built. + struct UnbuildableProvider(EchoProvider); + + impl IdentityProvider for UnbuildableProvider { + fn capabilities(&self) -> &ProviderCapabilities { + self.0.capabilities() + } + + fn build_request( + &self, + _lookup: &LookupInput<'_>, + ) -> Result> { + Err(Report::new(ProviderError::Request)) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + self.0.parse_response(response) + } + } + + #[test] + fn a_request_the_provider_cannot_build_is_reported_as_a_failure() { + let coordinator = ProviderCoordinator::new( + vec![RegisteredProvider::new( + provider_config(""), + UnbuildableProvider(echo_provider(SourceInput::Anonymous)), + )], + partners(), + Duration::from_millis(200), + ) + .expect("should build coordinator"); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &live_row(), &granting()); + + assert_eq!( + only_status(&results), + LookupStatus::RequestFailed, + "should report a provider bug instead of a skip" + ); + } + + /// An echo provider that generates its ID for the given source instead of + /// looking it up. + struct GeneratingProvider(EchoProvider, &'static str); + + impl IdentityProvider for GeneratingProvider { + fn capabilities(&self) -> &ProviderCapabilities { + self.0.capabilities() + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + self.0.build_request(lookup) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + self.0.parse_response(response) + } + + fn generate(&self, _lookup: &LookupInput<'_>) -> Option>> { + Some(vec![ProviderEid { + source: Cow::Borrowed(self.1), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids: vec![ProviderUid { + id: Cow::Borrowed("generated-1"), + atype: None, + ext: None, + expiry: ProviderExpiry::Unspecified, + refresh: None, + }], + }]) + } + } + + fn generating_coordinator() -> ProviderCoordinator { + generating_coordinator_for("ids.example.com") + } + + fn generating_coordinator_for(source: &'static str) -> ProviderCoordinator { + ProviderCoordinator::new( + vec![RegisteredProvider::new( + provider_config(""), + GeneratingProvider(echo_provider(SourceInput::Anonymous), source), + )], + partners(), + Duration::from_millis(200), + ) + .expect("should build coordinator") + } + + #[test] + fn a_generated_id_replaces_the_lookup_for_a_due_source() { + let coordinator = generating_coordinator(); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &live_row(), &granting()); + + let lookup = results.iter().next().expect("should report the provider"); + assert!( + matches!(lookup.status, LookupStatus::Found { kept: 1, .. }), + "should keep the generated UID" + ); + assert_eq!( + lookup.entries[0].1[0].uids()[0].id().as_str(), + "generated-1", + "should store the generated ID" + ); + assert!(transport.sent.borrow().is_empty(), "should send no lookup"); + } + + #[test] + fn a_generated_id_matches_its_due_source_after_normalizing() { + let coordinator = generating_coordinator_for("IDS.example.com"); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &live_row(), &granting()); + + let lookup = results.iter().next().expect("should report the provider"); + assert!( + matches!(lookup.status, LookupStatus::Found { kept: 1, .. }), + "should keep the generated UID for a differently cased source" + ); + assert!(transport.sent.borrow().is_empty(), "should send no lookup"); + } + + #[test] + fn a_refresh_due_live_source_is_looked_up_not_regenerated() { + let coordinator = generating_coordinator(); + let transport = CannedTransport::new(ok_reply); + + let results = run( + &coordinator, + &transport, + &snapshot_holding(refreshable_record(900)), + &granting(), + ); + + let lookup = results.iter().next().expect("should report the provider"); + assert!( + lookup + .entries + .iter() + .flat_map(|(_, eids)| eids) + .flat_map(KvEid::uids) + .all(|uid| uid.id().as_str() != "generated-1"), + "should not replace the live UID with a generated one" + ); + assert_eq!( + transport.sent.borrow().len(), + 1, + "should send the refresh lookup instead" + ); + } + + #[test] + fn a_held_source_is_not_regenerated() { + let coordinator = generating_coordinator(); + let transport = CannedTransport::new(ok_reply); + + let results = run( + &coordinator, + &transport, + &snapshot_holding(provider_record("p-1")), + &granting(), + ); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::NotDue), + "should keep the held ID" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/coordinator/results.rs b/crates/trusted-server-core/src/ec/coordinator/results.rs new file mode 100644 index 0000000000..f5015e4529 --- /dev/null +++ b/crates/trusted-server-core/src/ec/coordinator/results.rs @@ -0,0 +1,172 @@ +//! Per-provider outcomes of one navigation's lookups. + +use crate::ec::eid_types::{EidValidationError, ProviderId, SourceDomain}; +use crate::ec::kv_types::KvEid; + +/// Validation drops for one response, by reason. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct DropCounts { + /// Blank or oversized UIDs, or negative agent types. + pub invalid_uid: u16, + /// Invalid `ext`, `inserter`, `matcher` or `mm`. + pub invalid_field: u16, + /// Entries for sources the provider does not claim. + pub unclaimed_source: u16, + /// UIDs whose expiry is not after the request time. + pub expired: u16, + /// Entries or UIDs beyond the provider's shape limits. + pub over_limit: u16, + /// Duplicate UIDs within one source. + pub duplicate: u16, +} + +impl DropCounts { + /// Counts one validation failure. + pub fn record(&mut self, reason: EidValidationError) { + let counter = match reason { + EidValidationError::EmptyUid + | EidValidationError::UidTooLong + | EidValidationError::NegativeAgentType => &mut self.invalid_uid, + EidValidationError::UnknownMatchMethod + | EidValidationError::InvalidExt + | EidValidationError::FieldTooLong => &mut self.invalid_field, + EidValidationError::UnclaimedSource => &mut self.unclaimed_source, + EidValidationError::Expired => &mut self.expired, + }; + *counter = counter.saturating_add(1); + } + + /// Total drops. + #[must_use] + pub fn total(&self) -> u16 { + self.invalid_uid + .saturating_add(self.invalid_field) + .saturating_add(self.unclaimed_source) + .saturating_add(self.expired) + .saturating_add(self.over_limit) + .saturating_add(self.duplicate) + } +} + +/// Why a provider was not called. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SkipReason { + /// The EC has no readable row yet, so it may never be issued. + NoRow, + /// Consent does not permit this provider. + Consent, + /// None of its sources is due. + NotDue, + /// Every due source needs input V1 never has (hashed email). + Input, + /// The built URL is not `https` or not allowlisted. + Destination, + /// The provider's hourly rate limit is reached. + RateLimited, + /// The vendor reports an opt-out; the provider's stored UIDs are + /// removed. + OptedOut, +} + +/// One outcome per provider per navigation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LookupStatus { + /// Valid entries were returned. + Found { + /// UIDs kept after validation. + kept: u16, + /// UIDs or entries dropped by validation. + dropped: DropCounts, + }, + /// The provider knows no ID for this visitor. + NoMatch, + /// No response before the shared deadline. + Timeout, + /// A non-success HTTP status. + HttpError(http::StatusCode), + /// The provider could not build its request. + RequestFailed, + /// The request could not be sent or the body exceeded the limit. + TransportFailed, + /// The body could not be parsed at all. + InvalidResponse, + /// The provider was not called. + Skipped(SkipReason), +} + +impl LookupStatus { + /// Metric name for logs (spec §12.2). + #[must_use] + pub fn metric_name(&self) -> &'static str { + match self { + Self::Found { .. } => "found", + Self::NoMatch => "no_match", + Self::Timeout => "timeout", + Self::HttpError(_) => "http_error", + Self::RequestFailed => "request_error", + Self::TransportFailed => "transport_error", + Self::InvalidResponse => "invalid_response", + Self::Skipped(SkipReason::NoRow) => "skipped_no_row", + Self::Skipped(SkipReason::Consent) => "skipped_consent", + Self::Skipped(SkipReason::NotDue) => "skipped_not_due", + Self::Skipped(SkipReason::Input) => "skipped_input", + Self::Skipped(SkipReason::RateLimited) => "skipped_rate_limited", + Self::Skipped(SkipReason::Destination) => "destination_rejected", + Self::Skipped(SkipReason::OptedOut) => "skipped_opted_out", + } + } +} + +/// One provider's outcome and validated entries. +#[derive(Debug, Clone)] +pub struct ProviderLookup { + /// The provider. + pub provider: ProviderId, + /// What happened. + pub status: LookupStatus, + /// Validated entries per returned source; empty unless `Found`. + pub entries: Vec<(SourceDomain, Vec)>, + /// Sources whose stored records are removed, because the user opted + /// out of the provider; empty unless `Skipped(OptedOut)`. + pub cleared: Vec, +} + +/// Validated results of one navigation's lookups. +#[derive(Debug, Clone, Default)] +pub struct LookupResults { + /// EC ID the lookups were made for; `None` when nothing was looked up. + ec_id: Option, + lookups: Vec, +} + +impl LookupResults { + /// Wraps per-provider results, made for `ec_id`, in provider-config + /// order. + #[must_use] + pub fn new(ec_id: &str, lookups: Vec) -> Self { + Self { + ec_id: Some(ec_id.to_owned()), + lookups, + } + } + + /// Whether these results were looked up for `ec_id`. + /// + /// A navigation can switch EC IDs after dispatch (orphan recovery mints a + /// new one), so results for another ID must not be attributed to it. + #[must_use] + pub fn is_for(&self, ec_id: &str) -> bool { + self.ec_id.as_deref() == Some(ec_id) + } + + /// Iterates results in provider-config order. + pub fn iter(&self) -> impl Iterator { + self.lookups.iter() + } + + /// Whether no provider produced a result. + #[must_use] + pub fn is_empty(&self) -> bool { + self.lookups.is_empty() + } +} diff --git a/crates/trusted-server-core/src/ec/coordinator/transport.rs b/crates/trusted-server-core/src/ec/coordinator/transport.rs new file mode 100644 index 0000000000..668fbb5b91 --- /dev/null +++ b/crates/trusted-server-core/src/ec/coordinator/transport.rs @@ -0,0 +1,463 @@ +//! Sends provider lookups and gathers responses before a shared deadline. + +use std::cell::Cell; +use std::time::Duration; + +use edgezero_core::body::Body as EdgeBody; +use error_stack::{Report, ResultExt as _}; + +use crate::ec::provider::ProviderRequest; +use crate::platform::{ + PlatformBackendSpec, PlatformHttpRequest, PlatformPendingRequest, PlatformResponse, + RuntimeServices, +}; + +/// Largest provider response body accepted. +pub const MAX_PROVIDER_RESPONSE_BYTES: usize = 64 * 1024; + +/// Smallest transport timeout registered for a lookup backend, in +/// milliseconds; the shared deadline still bounds collection. +const MIN_TRANSPORT_TIMEOUT_MS: u32 = 50; + +/// Sends provider lookups and gathers their responses before a deadline. +#[allow(async_fn_in_trait)] +pub trait LookupTransport { + /// An in-flight request. + type Pending; + + /// Starts one request without waiting for its response. + /// + /// # Errors + /// + /// Returns [`TransportError::Send`] when the request cannot be started. + async fn start( + &self, + request: ProviderRequest, + deadline: Deadline, + ) -> Result>; + + /// Waits for in-flight requests until all finish or `deadline` passes. + /// + /// Requests still in flight at the deadline are abandoned and reported as + /// [`TransportError::Timeout`]. + async fn gather( + &self, + pending: Vec>, + deadline: Deadline, + ) -> Vec>>>; +} + +/// A value labelled with the provider it belongs to. +#[derive(Debug)] +pub struct Tagged { + /// The provider's position in the coordinator. + pub provider: ProviderIndex, + /// The labelled value. + pub value: T, +} + +/// Position of a provider in the coordinator's configured list. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct ProviderIndex(u16); + +impl ProviderIndex { + /// Wraps a position; the coordinator rejects more than `u16::MAX` providers. + /// + /// # Panics + /// + /// Panics when `index` does not fit in a `u16`. + pub(crate) fn new(index: usize) -> Self { + Self(u16::try_from(index).expect("should hold fewer than 65536 providers")) + } + + /// The position. + #[must_use] + pub fn get(self) -> usize { + usize::from(self.0) + } +} + +/// An owned response; providers parse it by borrowing its body. +#[derive(Debug)] +pub struct TransportResponse { + /// Response status. + pub status: http::StatusCode, + /// Response headers. + pub headers: http::HeaderMap, + /// Response body, at most [`MAX_PROVIDER_RESPONSE_BYTES`]. + pub body: bytes::Bytes, +} + +/// End of the shared lookup budget. +#[derive(Debug, Clone, Copy)] +pub struct Deadline(web_time::Instant); + +impl Deadline { + /// A deadline `budget` from now. + #[must_use] + pub fn after(budget: Duration) -> Self { + Self(web_time::Instant::now() + budget) + } + + /// The deadline instant. + #[must_use] + pub fn instant(self) -> web_time::Instant { + self.0 + } + + /// Time left before the deadline; zero once it has passed. + #[must_use] + pub fn remaining(self) -> Duration { + self.0.saturating_duration_since(web_time::Instant::now()) + } +} + +/// Why a lookup produced no response. +#[derive(Debug, derive_more::Display)] +pub enum TransportError { + /// The request could not be sent. + #[display("lookup request could not be sent")] + Send, + /// No response before the deadline. + #[display("lookup did not complete before the deadline")] + Timeout, + /// The body exceeded [`MAX_PROVIDER_RESPONSE_BYTES`]. + #[display("lookup response exceeded the body limit")] + BodyTooLarge, +} + +impl core::error::Error for TransportError {} + +/// [`LookupTransport`] over the platform HTTP client. +/// +/// Each request gets its own backend name (a per-transport sequence number is +/// the backend discriminator), so a ready response maps back to its provider +/// by backend name. Names repeat across requests and the transport timeout is +/// canonicalized, keeping the number of registered dynamic backends bounded. +pub struct PlatformLookupTransport<'s> { + services: &'s RuntimeServices, + next: Cell, +} + +impl<'s> PlatformLookupTransport<'s> { + /// Borrows the request's runtime services. + /// + /// Lookups always verify the provider's certificate: they carry the EC ID + /// and the provider's credential, so `proxy.certificate_check` (meant for + /// development origins) does not apply to them. + #[must_use] + pub fn new(services: &'s RuntimeServices) -> Self { + Self { + services, + next: Cell::new(0), + } + } + + fn transport_timeout(&self, deadline: Deadline) -> Duration { + let remaining_ms = u32::try_from(deadline.remaining().as_millis()).unwrap_or(u32::MAX); + let canonical = self + .services + .backend() + .canonicalize_transport_timeout_ms(remaining_ms, u32::MAX); + Duration::from_millis(u64::from(canonical.max(MIN_TRANSPORT_TIMEOUT_MS))) + } +} + +impl LookupTransport for PlatformLookupTransport<'_> { + type Pending = PlatformPendingRequest; + + async fn start( + &self, + request: ProviderRequest, + deadline: Deadline, + ) -> Result> { + let ProviderRequest { + method, + url, + headers, + body, + context: _, + } = request; + let host = url + .host_str() + .ok_or_else(|| Report::new(TransportError::Send))? + .to_owned(); + let timeout = self.transport_timeout(deadline); + let sequence = self.next.get(); + self.next.set(sequence.wrapping_add(1)); + let backend = self + .services + .backend() + .ensure(&PlatformBackendSpec { + scheme: url.scheme().to_owned(), + host, + port: url.port(), + host_header_override: None, + certificate_check: true, + first_byte_timeout: timeout, + between_bytes_timeout: timeout, + discriminator: Some(format!("ts-identity-{sequence}")), + }) + .change_context(TransportError::Send)?; + let mut builder = http::Request::builder().method(method).uri(url.as_str()); + if let Some(target) = builder.headers_mut() { + *target = headers; + } + let edge_request = builder + .body(EdgeBody::from(body.to_vec())) + .change_context(TransportError::Send)?; + self.services + .http_client() + .send_async(PlatformHttpRequest::new(edge_request, backend).with_cache_bypass()) + .await + .change_context(TransportError::Send) + } + + async fn gather( + &self, + pending: Vec>, + deadline: Deadline, + ) -> Vec>>> { + // Names are captured before the first select: some adapters drop + // them from the requests that remain in flight. + let mut outstanding: Vec<(String, ProviderIndex)> = Vec::with_capacity(pending.len()); + let mut remaining = Vec::with_capacity(pending.len()); + let mut results = Vec::with_capacity(pending.len()); + for Tagged { provider, value } in pending { + match value.backend_name() { + Some(name) => { + outstanding.push((name.to_owned(), provider)); + remaining.push(value); + } + None => results.push(Tagged { + provider, + value: Err(Report::new(TransportError::Send)), + }), + } + } + while !remaining.is_empty() { + match self + .services + .http_client() + .select_until(remaining, deadline.instant()) + .await + { + Ok(Some(selected)) => { + remaining = selected.remaining; + let (name, value) = match selected.ready { + Ok(response) => { + (response.backend_name.clone(), read_response(response).await) + } + Err(report) => ( + selected.failed_backend_name, + Err(report.change_context(TransportError::Send)), + ), + }; + match name.and_then(|name| take_provider(&mut outstanding, &name)) { + Some(provider) => results.push(Tagged { provider, value }), + None => { + log::warn!("Identity lookup: dropping response with unknown backend"); + } + } + } + Ok(None) => break, + Err(report) => { + log::warn!("Identity lookup: select failed: {report:?}"); + break; + } + } + } + // Every request not yet answered timed out or was abandoned. + results.extend(outstanding.into_iter().map(|(_, provider)| Tagged { + provider, + value: Err(Report::new(TransportError::Timeout)), + })); + results + } +} + +/// Removes the first outstanding request with backend `name`. +fn take_provider( + outstanding: &mut Vec<(String, ProviderIndex)>, + name: &str, +) -> Option { + let index = outstanding + .iter() + .position(|(candidate, _)| candidate == name)?; + Some(outstanding.remove(index).1) +} + +async fn read_response( + response: PlatformResponse, +) -> Result> { + let (parts, body) = response.response.into_parts(); + let body = match body { + EdgeBody::Once(bytes) => bytes, + stream => stream + .into_bytes_bounded(MAX_PROVIDER_RESPONSE_BYTES) + .await + .change_context(TransportError::BodyTooLarge)?, + }; + if body.len() > MAX_PROVIDER_RESPONSE_BYTES { + return Err(Report::new(TransportError::BodyTooLarge)); + } + Ok(TransportResponse { + status: parts.status, + headers: parts.headers, + body, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::platform::test_support::{StubHttpClient, build_services_with_http_client}; + use std::sync::Arc; + + fn request(path: &str) -> ProviderRequest { + ProviderRequest { + method: http::Method::GET, + url: url::Url::parse(&format!("https://ids.example.com/{path}")).expect("should parse"), + headers: http::HeaderMap::new(), + body: bytes::Bytes::new(), + context: None, + } + } + + fn gather_one( + transport: &PlatformLookupTransport<'_>, + deadline: Deadline, + ) -> Vec>>> { + futures::executor::block_on(async { + let pending = transport + .start(request("a"), deadline) + .await + .expect("should start"); + transport + .gather( + vec![Tagged { + provider: ProviderIndex::new(0), + value: pending, + }], + deadline, + ) + .await + }) + } + + #[test] + fn gather_returns_tagged_responses() { + let client = Arc::new(StubHttpClient::new()); + client.push_response(200, b"{\"a\":1}".to_vec()); + client.push_response(204, Vec::new()); + let services = build_services_with_http_client(client.clone()); + let transport = PlatformLookupTransport::new(&services); + let deadline = Deadline::after(Duration::from_millis(200)); + + let results = futures::executor::block_on(async { + let first = transport + .start(request("a"), deadline) + .await + .expect("should start"); + let second = transport + .start(request("b"), deadline) + .await + .expect("should start"); + transport + .gather( + vec![ + Tagged { + provider: ProviderIndex::new(0), + value: first, + }, + Tagged { + provider: ProviderIndex::new(1), + value: second, + }, + ], + deadline, + ) + .await + }); + + assert_eq!(results.len(), 2, "should return one result per request"); + let first = results + .iter() + .find(|result| result.provider == ProviderIndex::new(0)) + .expect("should tag provider 0"); + let first = first.value.as_ref().expect("should succeed"); + assert_eq!(first.status, http::StatusCode::OK, "should map provider 0"); + assert_eq!(first.body.as_ref(), b"{\"a\":1}", "should carry the body"); + let second = results + .iter() + .find(|result| result.provider == ProviderIndex::new(1)) + .expect("should tag provider 1"); + assert_eq!( + second.value.as_ref().expect("should succeed").status, + http::StatusCode::NO_CONTENT, + "should map provider 1" + ); + assert_eq!( + client.recorded_request_uris(), + [ + "https://ids.example.com/a".to_owned(), + "https://ids.example.com/b".to_owned() + ], + "should send both requests" + ); + } + + #[test] + fn gather_reports_timeout_for_requests_past_the_deadline() { + let client = Arc::new(StubHttpClient::new()); + client.push_response(200, Vec::new()); + client.push_select_delay(Duration::from_millis(50)); + let services = build_services_with_http_client(client.clone()); + let transport = PlatformLookupTransport::new(&services); + + let results = gather_one(&transport, Deadline::after(Duration::from_millis(1))); + + let error = results[0].value.as_ref().expect_err("should time out"); + assert!( + matches!(error.current_context(), TransportError::Timeout), + "should report timeout" + ); + } + + #[test] + fn gather_rejects_oversized_bodies() { + let client = Arc::new(StubHttpClient::new()); + client.push_response(200, vec![b'x'; MAX_PROVIDER_RESPONSE_BYTES + 1]); + let services = build_services_with_http_client(client.clone()); + let transport = PlatformLookupTransport::new(&services); + + let results = gather_one(&transport, Deadline::after(Duration::from_millis(200))); + + let error = results[0] + .value + .as_ref() + .expect_err("should reject the body"); + assert!( + matches!(error.current_context(), TransportError::BodyTooLarge), + "should report an oversized body" + ); + } + + #[test] + fn gather_with_no_requests_returns_nothing() { + let client = Arc::new(StubHttpClient::new()); + let services = build_services_with_http_client(client.clone()); + let transport = PlatformLookupTransport::new(&services); + + let results = futures::executor::block_on( + transport.gather(Vec::new(), Deadline::after(Duration::from_millis(1))), + ); + + assert!(results.is_empty(), "should return nothing"); + assert_eq!( + client.recorded_select_count(), + 0, + "should not select an empty list" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/coordinator/validate.rs b/crates/trusted-server-core/src/ec/coordinator/validate.rs new file mode 100644 index 0000000000..9d9a63b175 --- /dev/null +++ b/crates/trusted-server-core/src/ec/coordinator/validate.rs @@ -0,0 +1,437 @@ +//! Turns a provider's parsed response into stored entries (spec §8.2). + +use std::borrow::Cow; + +use crate::ec::coordinator::results::DropCounts; +use crate::ec::eid_types::{ + AgentType, EidFields, EidValidationError, Expiry, MAX_EID_FIELD_BYTES, MatchMethod, ProviderId, + RawJson, SourceDomain, UidValue, UnixSeconds, +}; +use crate::ec::kv_types::{KvEid, KvRefresh, KvUid}; +use crate::ec::provider::{EidLevelFields, EidShape, ProviderEid, ProviderExpiry, ProviderUid}; +use crate::ec::registry::PartnerRegistry; + +/// Largest private refresh state a provider may keep with one UID. +const MAX_REFRESH_STATE_BYTES: usize = 2048; + +/// Policy one response is validated against. +pub(crate) struct ValidationPolicy<'a> { + /// Owner recorded on every kept UID. + pub provider: &'a ProviderId, + /// Sources the provider claims in config. + pub claimed: &'a [SourceDomain], + /// The provider's declared shape limits. + pub shape: EidShape, + /// Partner config, for the default `atype`. + pub partners: &'a PartnerRegistry, + /// Request time. + pub now: UnixSeconds, +} + +/// Validated entries grouped by source, in response order. +pub(crate) type ValidatedEntries = Vec<(SourceDomain, Vec)>; + +/// Validates parsed entries, keeping valid ones grouped by source in +/// response order. +pub(crate) fn validate( + eids: Vec>, + policy: &ValidationPolicy<'_>, +) -> (ValidatedEntries, DropCounts) { + let mut dropped = DropCounts::default(); + let mut grouped: ValidatedEntries = Vec::new(); + let mut kept_eids = 0_usize; + + for eid in eids { + let Some(source) = claimed_source(&eid.source, policy.claimed) else { + dropped.record(EidValidationError::UnclaimedSource); + continue; + }; + if kept_eids == policy.shape.max_eids.get() { + dropped.over_limit = dropped.over_limit.saturating_add(1); + continue; + } + let fields = match eid_fields(&eid, policy.shape.eid_level_fields) { + Ok(fields) => fields, + Err(reason) => { + dropped.record(reason); + continue; + } + }; + let default_atype = policy + .partners + .find_by_source_domain(source.as_str()) + .map_or(1, |partner| partner.openrtb_atype); + let slot = match grouped.iter().position(|(existing, _)| existing == source) { + Some(index) => index, + None => { + grouped.push((source.clone(), Vec::new())); + grouped.len() - 1 + } + }; + let max_uids = policy.shape.max_uids_per_eid.get(); + let mut uids = Vec::with_capacity(eid.uids.len().min(max_uids)); + for uid in eid.uids { + if uids.len() == max_uids { + dropped.over_limit = dropped.over_limit.saturating_add(1); + continue; + } + match stored_uid(uid, default_atype, policy) { + Ok(uid) if is_duplicate(&grouped[slot].1, &uids, &uid) => { + dropped.duplicate = dropped.duplicate.saturating_add(1); + } + Ok(uid) => uids.push(uid), + Err(reason) => dropped.record(reason), + } + } + if !uids.is_empty() { + grouped[slot].1.push(KvEid::new(fields, uids)); + kept_eids += 1; + } + } + + grouped.retain(|(_, eids)| !eids.is_empty()); + (grouped, dropped) +} + +fn claimed_source<'p>(raw: &str, claimed: &'p [SourceDomain]) -> Option<&'p SourceDomain> { + let source = SourceDomain::try_from(raw).ok()?; + claimed.iter().find(|candidate| **candidate == source) +} + +fn bounded_field(value: Option<&Cow<'_, str>>) -> Result>, EidValidationError> { + match value { + Some(text) if text.len() > MAX_EID_FIELD_BYTES => Err(EidValidationError::FieldTooLong), + Some(text) => Ok(Some(Box::from(text.as_ref()))), + None => Ok(None), + } +} + +fn eid_fields( + eid: &ProviderEid<'_>, + level: EidLevelFields, +) -> Result { + if level == EidLevelFields::Ignored { + return Ok(EidFields::default()); + } + Ok(EidFields { + inserter: bounded_field(eid.inserter.as_ref())?, + matcher: bounded_field(eid.matcher.as_ref())?, + mm: eid.mm.map(MatchMethod::try_from).transpose()?, + ext: eid.ext.map(RawJson::try_from).transpose()?, + }) +} + +fn stored_uid( + uid: ProviderUid<'_>, + default_atype: i32, + policy: &ValidationPolicy<'_>, +) -> Result { + let id = UidValue::try_from(uid.id.into_owned())?; + let atype = AgentType::try_from(uid.atype.unwrap_or(default_atype))?; + let ext = uid.ext.map(RawJson::try_from).transpose()?; + let expiry = match resolve_expiry(uid.expiry, policy.now) { + Some(at) => Expiry::At(at), + None => Expiry::Never, + }; + if !expiry.is_live(policy.now) { + return Err(EidValidationError::Expired); + } + let stored = KvUid::new(id, atype, ext, Some(policy.provider.clone()), expiry); + let Some(refresh) = uid.refresh else { + return Ok(stored); + }; + // Unusable refresh state is dropped, never the UID: a refresh with no + // stated time, with empty or oversized state, or due already (which + // would make the source due on every navigation) is simply not stored. + let usable = !refresh.state.is_empty() && refresh.state.len() <= MAX_REFRESH_STATE_BYTES; + match resolve_expiry(refresh.from, policy.now) { + Some(from) if usable && from > policy.now => Ok(stored.with_refresh(KvRefresh { + state: refresh.state.into_owned(), + from, + })), + _ => { + log::debug!( + "Identity lookup: provider={} drops unusable refresh state", + policy.provider + ); + Ok(stored) + } + } +} + +/// Resolves a provider-stated time against `now`. `None` when no time was +/// stated. +fn resolve_expiry(expiry: ProviderExpiry, now: UnixSeconds) -> Option { + match expiry { + ProviderExpiry::Unspecified => None, + ProviderExpiry::At(at) => Some(at), + ProviderExpiry::After(after) => Some(now.saturating_add_secs(after.as_secs())), + } +} + +fn is_duplicate(earlier: &[KvEid], current: &[KvUid], uid: &KvUid) -> bool { + earlier + .iter() + .flat_map(KvEid::uids) + .chain(current) + .any(|existing| existing.id() == uid.id()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ec::provider::ProviderRefresh; + use crate::settings::EcPartner; + use serde_json::value::RawValue; + use std::num::NonZeroUsize; + use std::time::Duration; + + fn registry() -> PartnerRegistry { + let partner: EcPartner = serde_json::from_value(serde_json::json!({ + "name": "ids", "source_domain": "ids.example.com", "openrtb_atype": 3, "bidstream_enabled": true + })) + .expect("should build partner"); + PartnerRegistry::from_config(&[partner]).expect("should build registry") + } + + fn uid(id: &str) -> ProviderUid<'_> { + ProviderUid { + id: Cow::Borrowed(id), + atype: None, + ext: None, + expiry: ProviderExpiry::Unspecified, + refresh: None, + } + } + + fn eid<'a>(source: &'a str, uids: Vec>) -> ProviderEid<'a> { + ProviderEid { + source: Cow::Borrowed(source), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids, + } + } + + fn policy<'a>( + provider: &'a ProviderId, + claimed: &'a [SourceDomain], + partners: &'a PartnerRegistry, + ) -> ValidationPolicy<'a> { + ValidationPolicy { + provider, + claimed, + shape: EidShape { + max_eids: NonZeroUsize::new(2).expect("should be non-zero"), + max_uids_per_eid: NonZeroUsize::new(2).expect("should be non-zero"), + eid_level_fields: EidLevelFields::Preserved, + }, + partners, + now: UnixSeconds::new(1000), + } + } + + fn claimed() -> [SourceDomain; 1] { + [SourceDomain::try_from("ids.example.com").expect("should parse")] + } + + #[test] + fn keeps_valid_uids_and_fills_owner_atype_and_expiry() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + let mut timed = uid("u-2"); + timed.expiry = ProviderExpiry::After(Duration::from_secs(60)); + + let (entries, dropped) = validate( + vec![eid("IDS.example.com", vec![uid("u-1"), timed])], + &policy(&provider, &claimed, &partners), + ); + + assert_eq!(dropped, DropCounts::default(), "should drop nothing"); + let uids = entries[0].1[0].uids(); + assert_eq!(uids[0].atype().get(), 3, "should fill atype from partner"); + assert_eq!(uids[0].owner(), Some(&provider), "should record owner"); + assert_eq!( + uids[1].expiry(), + Expiry::At(UnixSeconds::new(1060)), + "should resolve relative expiry" + ); + } + + #[test] + fn keeps_the_uid_when_its_refresh_state_is_unusable() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + let in_a_minute = ProviderExpiry::After(Duration::from_secs(60)); + let cases = [ + ( + "state".to_owned(), + in_a_minute, + Some(UnixSeconds::new(1060)), + ), + (String::new(), in_a_minute, None), + ("x".repeat(MAX_REFRESH_STATE_BYTES + 1), in_a_minute, None), + ("state".to_owned(), ProviderExpiry::Unspecified, None), + ( + "state".to_owned(), + ProviderExpiry::After(Duration::ZERO), + None, + ), + ( + "state".to_owned(), + ProviderExpiry::At(UnixSeconds::new(1000)), + None, + ), + ( + "state".to_owned(), + ProviderExpiry::At(UnixSeconds::new(999)), + None, + ), + ]; + + for (state, from, expected) in cases { + let mut refreshed = uid("u-1"); + refreshed.refresh = Some(ProviderRefresh { + state: Cow::Owned(state), + from, + }); + + let (entries, dropped) = validate( + vec![eid("ids.example.com", vec![refreshed])], + &policy(&provider, &claimed, &partners), + ); + + assert_eq!(dropped, DropCounts::default(), "should drop no UID"); + assert_eq!( + entries[0].1[0].uids()[0] + .refresh() + .map(|refresh| refresh.from), + expected, + "should keep only usable refresh state" + ); + } + } + + #[test] + fn drops_invalid_values_by_reason() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + let long = "x".repeat(crate::ec::kv_types::MAX_STORED_UID_LENGTH + 1); + let mut expired = uid("u-old"); + expired.expiry = ProviderExpiry::At(UnixSeconds::new(1000)); + let array_ext = RawValue::from_string("[1]".to_owned()).expect("should build raw"); + let mut bad_ext = uid("u-ext"); + bad_ext.ext = Some(&array_ext); + + let (entries, dropped) = validate( + vec![ + eid( + "ids.example.com", + vec![uid(" "), uid(&long), expired, bad_ext, uid("ok"), uid("ok")], + ), + eid("unclaimed.example.com", vec![uid("u-3")]), + ], + &policy(&provider, &claimed, &partners), + ); + + assert_eq!(dropped.invalid_uid, 2, "should drop blank and long UIDs"); + assert_eq!(dropped.expired, 1, "should drop expired UID"); + assert_eq!( + dropped.invalid_field, 1, + "should drop UID with non-object ext" + ); + assert_eq!(dropped.duplicate, 1, "should drop duplicate UID"); + assert_eq!(dropped.unclaimed_source, 1, "should drop unclaimed source"); + assert_eq!(entries.len(), 1, "should keep the claimed source only"); + assert_eq!(entries[0].1[0].uids().len(), 1, "should keep one valid UID"); + } + + #[test] + fn enforces_shape_limits_from_the_end() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + + let (entries, dropped) = validate( + vec![ + eid("ids.example.com", vec![uid("a"), uid("b"), uid("c")]), + eid("ids.example.com", vec![uid("d")]), + eid("ids.example.com", vec![uid("e")]), + ], + &policy(&provider, &claimed, &partners), + ); + + assert_eq!( + dropped.over_limit, 2, + "should drop third entry and third UID" + ); + let ids: Vec<&str> = entries[0] + .1 + .iter() + .flat_map(KvEid::uids) + .map(|uid| uid.id().as_str()) + .collect(); + assert_eq!( + ids, + ["a", "b", "d"], + "should keep the first entries and UIDs" + ); + } + + #[test] + fn ignored_eid_level_fields_are_stripped_not_rejected() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + let mut with_fields = eid("ids.example.com", vec![uid("a")]); + with_fields.inserter = Some(Cow::Borrowed("inserter.example.com")); + with_fields.mm = Some(99); + let mut policy = policy(&provider, &claimed, &partners); + policy.shape.eid_level_fields = EidLevelFields::Ignored; + + let (entries, dropped) = validate(vec![with_fields], &policy); + + assert_eq!( + dropped, + DropCounts::default(), + "should not count ignored fields" + ); + assert_eq!( + entries[0].1[0].fields(), + &EidFields::default(), + "should strip EID-level fields" + ); + } + + #[test] + fn preserved_eid_level_fields_are_validated() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + let mut bad_mm = eid("ids.example.com", vec![uid("a")]); + bad_mm.mm = Some(99); + let mut kept = eid("ids.example.com", vec![uid("b")]); + kept.inserter = Some(Cow::Borrowed("inserter.example.com")); + kept.mm = Some(3); + + let (entries, dropped) = + validate(vec![bad_mm, kept], &policy(&provider, &claimed, &partners)); + + assert_eq!(dropped.invalid_field, 1, "should drop unknown mm entry"); + assert_eq!( + entries[0].1[0].fields().inserter.as_deref(), + Some("inserter.example.com"), + "should keep valid inserter" + ); + assert_eq!( + entries[0].1[0].fields().mm, + Some(MatchMethod::Authenticated), + "should keep valid mm" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/eid_types.rs b/crates/trusted-server-core/src/ec/eid_types.rs new file mode 100644 index 0000000000..74b54aca88 --- /dev/null +++ b/crates/trusted-server-core/src/ec/eid_types.rs @@ -0,0 +1,678 @@ +//! Value types for full-fidelity EIDs stored in the EC identity graph. +//! +//! Provider output is converted into these types at the trust boundary +//! (coordinator validation); past that point an invalid UID, agent type or +//! match method cannot be represented. + +use core::fmt; +use std::sync::Arc; + +use serde::de::{self, Deserializer, Visitor}; +use serde::{Deserialize, Serialize, Serializer}; +use serde_json::value::RawValue; + +use super::kv_types::MAX_STORED_UID_LENGTH; +use super::partner::normalize_partner_source_domain; + +/// Maximum serialized size of an `ext` object, in bytes. +pub const MAX_EXT_BYTES: usize = 1024; + +/// Maximum nesting depth of an `ext` object. +/// +/// Keeps every whole-row or whole-request parse that embeds the `ext` well +/// under `serde_json`'s 128-level recursion limit. +pub const MAX_EXT_DEPTH: usize = 32; + +/// Returns the deepest object or array nesting in JSON `text`. +/// +/// Iterative, so it cannot overflow the stack on hostile input. +fn json_depth(text: &str) -> usize { + let (mut depth, mut deepest) = (0_usize, 0_usize); + let (mut in_string, mut escaped) = (false, false); + for byte in text.bytes() { + if in_string { + match byte { + _ if escaped => escaped = false, + b'\\' => escaped = true, + b'"' => in_string = false, + _ => {} + } + continue; + } + match byte { + b'"' => in_string = true, + b'{' | b'[' => { + depth += 1; + deepest = deepest.max(depth); + } + b'}' | b']' => depth = depth.saturating_sub(1), + _ => {} + } + } + deepest +} + +/// Maximum size of `inserter` and `matcher`, in bytes. +pub const MAX_EID_FIELD_BYTES: usize = 255; + +/// Why a provider value was rejected; counted per reason, never logged with +/// the value. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub enum EidValidationError { + /// The UID is blank. + #[display("UID is empty")] + EmptyUid, + /// The UID exceeds [`MAX_STORED_UID_LENGTH`]. + #[display("UID is too long")] + UidTooLong, + /// The agent type is negative. + #[display("agent type is negative")] + NegativeAgentType, + /// The match method is not an `OpenRTB` 2.6 value. + #[display("match method is unknown")] + UnknownMatchMethod, + /// An `ext` is not a JSON object within [`MAX_EXT_BYTES`]. + #[display("ext is not a JSON object within the size limit")] + InvalidExt, + /// An `inserter` or `matcher` exceeds [`MAX_EID_FIELD_BYTES`]. + #[display("EID field is too long")] + FieldTooLong, + /// The source is not a valid domain or not claimed by the provider. + #[display("source is not claimed by this provider")] + UnclaimedSource, + /// The expiry is not after the request time. + #[display("expiry is in the past")] + Expired, +} + +impl core::error::Error for EidValidationError {} + +/// Unix time in whole seconds. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + derive_more::Display, +)] +#[serde(transparent)] +pub struct UnixSeconds(u64); + +impl UnixSeconds { + /// Wraps a Unix timestamp in seconds. + #[must_use] + pub const fn new(seconds: u64) -> Self { + Self(seconds) + } + + /// Returns the timestamp in seconds. + #[must_use] + pub const fn get(self) -> u64 { + self.0 + } + + /// Adds `seconds`, saturating at `u64::MAX`. + #[must_use] + pub const fn saturating_add_secs(self, seconds: u64) -> Self { + Self(self.0.saturating_add(seconds)) + } +} + +struct ArcStrVisitor; + +impl<'de> Visitor<'de> for ArcStrVisitor { + type Value = Arc; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a string") + } + + fn visit_str(self, value: &str) -> Result { + Ok(Arc::from(value)) + } +} + +/// Operator-chosen identifier of a configured provider instance. +/// +/// Clones share one allocation. Serialized as a plain string. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, derive_more::Display)] +pub struct ProviderId(Arc); + +impl ProviderId { + /// Creates a provider ID from configuration. + #[must_use] + pub fn new(id: &str) -> Self { + Self(Arc::from(id)) + } + + /// Returns the ID as a string slice. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl Serialize for ProviderId { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0) + } +} + +impl<'de> Deserialize<'de> for ProviderId { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_str(ArcStrVisitor).map(Self) + } +} + +/// Normalized EID source domain. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, derive_more::Display)] +pub struct SourceDomain(Arc); + +impl SourceDomain { + /// Returns the normalized domain. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl TryFrom<&str> for SourceDomain { + type Error = EidValidationError; + + fn try_from(value: &str) -> Result { + normalize_partner_source_domain(value) + .map(|normalized| Self(Arc::from(normalized))) + .map_err(|_| EidValidationError::UnclaimedSource) + } +} + +impl Serialize for SourceDomain { + fn serialize(&self, serializer: S) -> Result { + serializer.serialize_str(&self.0) + } +} + +impl<'de> Deserialize<'de> for SourceDomain { + fn deserialize>(deserializer: D) -> Result { + let raw = deserializer.deserialize_str(ArcStrVisitor)?; + Self::try_from(raw.as_ref()).map_err(de::Error::custom) + } +} + +/// A non-blank UID of at most [`MAX_STORED_UID_LENGTH`] bytes, kept verbatim. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, derive_more::Display)] +#[serde(try_from = "String")] +pub struct UidValue(Box); + +impl UidValue { + /// Returns the UID. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } + + fn check(value: &str) -> Result<(), EidValidationError> { + if value.trim().is_empty() { + return Err(EidValidationError::EmptyUid); + } + if value.len() > MAX_STORED_UID_LENGTH { + return Err(EidValidationError::UidTooLong); + } + Ok(()) + } +} + +impl TryFrom<&str> for UidValue { + type Error = EidValidationError; + + fn try_from(value: &str) -> Result { + Self::check(value)?; + Ok(Self(Box::from(value))) + } +} + +impl TryFrom for UidValue { + type Error = EidValidationError; + + fn try_from(value: String) -> Result { + Self::check(&value)?; + Ok(Self(value.into_boxed_str())) + } +} + +/// Non-negative `OpenRTB` agent type, including vendor-specific values. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(try_from = "i32", into = "i32")] +pub struct AgentType(i32); + +impl AgentType { + /// Returns the raw agent type. + #[must_use] + pub const fn get(self) -> i32 { + self.0 + } +} + +impl TryFrom for AgentType { + type Error = EidValidationError; + + fn try_from(value: i32) -> Result { + if value < 0 { + return Err(EidValidationError::NegativeAgentType); + } + Ok(Self(value)) + } +} + +impl From for i32 { + fn from(value: AgentType) -> Self { + value.0 + } +} + +/// `OpenRTB` 2.6 EID match method (`mm`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(try_from = "i32", into = "i32")] +pub enum MatchMethod { + /// `0`: unknown. + Unknown, + /// `1`: no matching. + NoMatching, + /// `2`: cookie sync. + CookieSync, + /// `3`: authenticated. + Authenticated, + /// `4`: observed. + Observed, + /// `5`: inference. + Inference, +} + +impl TryFrom for MatchMethod { + type Error = EidValidationError; + + fn try_from(value: i32) -> Result { + match value { + 0 => Ok(Self::Unknown), + 1 => Ok(Self::NoMatching), + 2 => Ok(Self::CookieSync), + 3 => Ok(Self::Authenticated), + 4 => Ok(Self::Observed), + 5 => Ok(Self::Inference), + _ => Err(EidValidationError::UnknownMatchMethod), + } + } +} + +impl From for i32 { + fn from(value: MatchMethod) -> Self { + match value { + MatchMethod::Unknown => 0, + MatchMethod::NoMatching => 1, + MatchMethod::CookieSync => 2, + MatchMethod::Authenticated => 3, + MatchMethod::Observed => 4, + MatchMethod::Inference => 5, + } + } +} + +/// When a stored UID stops being served. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Expiry { + /// Served until replaced or removed. + #[default] + Never, + /// Never served at or after this time. + At(UnixSeconds), +} + +impl Expiry { + /// Returns whether a UID with this expiry may be served at `now`. + #[must_use] + pub fn is_live(self, now: UnixSeconds) -> bool { + match self { + Self::Never => true, + Self::At(expires_at) => now < expires_at, + } + } + + /// Returns whether this is [`Expiry::Never`]; used to omit `exp`. + #[must_use] + pub fn is_never(&self) -> bool { + matches!(self, Self::Never) + } +} + +/// Serde adapter storing [`Expiry`] as an optional `exp` number. +pub mod optional_exp { + use serde::{Deserialize as _, Deserializer, Serialize as _, Serializer}; + + use super::{Expiry, UnixSeconds}; + + /// Serializes `At` as its seconds; `Never` is skipped by the caller. + /// + /// # Errors + /// + /// Returns the serializer's error. + pub fn serialize(expiry: &Expiry, serializer: S) -> Result { + match expiry { + Expiry::At(expires_at) => expires_at.serialize(serializer), + Expiry::Never => serializer.serialize_none(), + } + } + + /// Reads an optional number; absent or `null` means [`Expiry::Never`]. + /// + /// # Errors + /// + /// Returns the deserializer's error for a non-numeric value. + pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result { + Ok(Option::::deserialize(deserializer)?.map_or(Expiry::Never, Expiry::At)) + } +} + +/// An opaque JSON object of at most [`MAX_EXT_BYTES`], compared by bytes. +/// +/// Deserializing applies the same checks as [`RawJson::try_from`], so a +/// stored row cannot carry an oversized or non-object `ext`. +#[derive(Debug, Clone)] +pub struct RawJson(Box); + +impl RawJson { + /// Returns the raw JSON text. + #[must_use] + pub fn as_str(&self) -> &str { + self.0.get() + } + + /// Parses the raw JSON into a [`serde_json::Value`] for `OpenRTB` output. + /// + /// # Panics + /// + /// Never in practice: the bytes were validated as JSON when stored. + #[must_use] + pub fn to_value(&self) -> serde_json::Value { + serde_json::from_str(self.0.get()).expect("should parse stored raw JSON") + } +} + +impl PartialEq for RawJson { + fn eq(&self, other: &Self) -> bool { + self.0.get() == other.0.get() + } +} + +impl RawJson { + fn check(value: &RawValue) -> Result<(), EidValidationError> { + let text = value.get(); + if text.len() > MAX_EXT_BYTES + || !text.trim_start().starts_with('{') + || json_depth(text) > MAX_EXT_DEPTH + { + return Err(EidValidationError::InvalidExt); + } + // `RawValue` skips nested values without serde_json's recursion limit, + // so parse once here: an `ext` that cannot become a `Value` would make + // `to_value` panic on every serve of the row. + serde_json::from_str::(text) + .map(|_| ()) + .map_err(|_| EidValidationError::InvalidExt) + } +} + +impl TryFrom<&RawValue> for RawJson { + type Error = EidValidationError; + + fn try_from(value: &RawValue) -> Result { + Self::check(value)?; + Ok(Self(value.to_owned())) + } +} + +impl TryFrom> for RawJson { + type Error = EidValidationError; + + fn try_from(value: Box) -> Result { + Self::check(&value)?; + Ok(Self(value)) + } +} + +impl Serialize for RawJson { + fn serialize(&self, serializer: S) -> Result { + self.0.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for RawJson { + fn deserialize>(deserializer: D) -> Result { + let raw = Box::::deserialize(deserializer)?; + Self::try_from(raw).map_err(de::Error::custom) + } +} + +/// EID-level fields shared by all UIDs of one stored entry. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +pub struct EidFields { + /// `OpenRTB` `inserter`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub inserter: Option>, + /// `OpenRTB` `matcher`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub matcher: Option>, + /// `OpenRTB` `mm`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mm: Option, + /// `OpenRTB` EID `ext`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ext: Option, +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn uid_value_rejects_blank_and_oversized() { + assert!(UidValue::try_from(" ").is_err(), "should reject blank UID"); + let long = "a".repeat(MAX_STORED_UID_LENGTH + 1); + assert!( + UidValue::try_from(long.as_str()).is_err(), + "should reject oversized UID" + ); + let uid = UidValue::try_from("ID-123").expect("should accept valid UID"); + assert_eq!(uid.as_str(), "ID-123", "should keep UID verbatim"); + } + + #[test] + fn agent_type_rejects_negative() { + assert!( + AgentType::try_from(-1).is_err(), + "should reject negative atype" + ); + assert_eq!( + AgentType::try_from(571_187) + .expect("should accept vendor atype") + .get(), + 571_187, + "should keep vendor atype" + ); + } + + #[test] + fn match_method_round_trips_openrtb_values() { + for value in 0..=5 { + let method = MatchMethod::try_from(value).expect("should accept OpenRTB mm"); + assert_eq!(i32::from(method), value, "should round-trip mm {value}"); + } + assert!( + MatchMethod::try_from(6).is_err(), + "should reject unknown mm" + ); + } + + #[test] + fn source_domain_normalizes() { + let source = SourceDomain::try_from("IDS.Example.COM").expect("should accept domain"); + assert_eq!( + source.as_str(), + "ids.example.com", + "should lowercase source domain" + ); + } + + #[test] + fn provider_id_round_trips_as_string() { + let id = ProviderId::new("example-provider"); + let encoded = serde_json::to_string(&id).expect("should serialize provider id"); + assert_eq!( + encoded, "\"example-provider\"", + "should serialize as plain string" + ); + let decoded: ProviderId = + serde_json::from_str(&encoded).expect("should deserialize provider id"); + assert_eq!(decoded, id, "should round-trip provider id"); + } + + #[test] + fn expiry_liveness() { + let now = UnixSeconds::new(100); + assert!( + Expiry::Never.is_live(now), + "should serve UIDs without expiry" + ); + assert!( + Expiry::At(UnixSeconds::new(101)).is_live(now), + "should serve before expiry" + ); + assert!( + !Expiry::At(UnixSeconds::new(100)).is_live(now), + "should not serve at expiry" + ); + } + + #[test] + fn raw_json_accepts_small_objects_only() { + let object = + serde_json::value::to_raw_value(&json!({ "k": 1 })).expect("should encode object"); + let raw = RawJson::try_from(object.as_ref()).expect("should accept object"); + assert_eq!(raw.as_str(), r#"{"k":1}"#, "should keep raw bytes"); + let array = serde_json::value::to_raw_value(&json!([1])).expect("should encode array"); + assert!( + RawJson::try_from(array.as_ref()).is_err(), + "should reject non-object" + ); + let big = serde_json::value::to_raw_value(&json!({ "k": "x".repeat(MAX_EXT_BYTES) })) + .expect("should encode big object"); + assert!( + RawJson::try_from(big.as_ref()).is_err(), + "should reject oversized ext" + ); + } + + #[test] + fn raw_json_rejects_ext_nested_past_the_parse_limit() { + let depth = 150; + let text = format!("{}1{}", r#"{"a":"#.repeat(depth), "}".repeat(depth)); + let raw: Box = + serde_json::from_str(&text).expect("should read deep JSON as a raw value"); + assert!( + text.len() <= MAX_EXT_BYTES, + "should fit the size limit so only the depth is under test" + ); + + assert!( + RawJson::try_from(raw).is_err(), + "should reject an ext that cannot be parsed into a value" + ); + } + + #[test] + fn raw_json_bounds_ext_nesting_depth() { + let nested = |depth: usize| -> Box { + let text = format!("{}1{}", r#"{"a":"#.repeat(depth), "}".repeat(depth)); + serde_json::from_str(&text).expect("should read JSON as a raw value") + }; + + assert!( + RawJson::try_from(nested(MAX_EXT_DEPTH)).is_ok(), + "should accept an ext at the depth limit" + ); + assert!( + RawJson::try_from(nested(MAX_EXT_DEPTH + 1)).is_err(), + "should reject an ext past the depth limit" + ); + let braces_in_string: Box = + serde_json::from_str(&format!(r#"{{"a":"{}\""}}"#, "{[".repeat(50))) + .expect("should read JSON as a raw value"); + assert!( + RawJson::try_from(braces_in_string).is_ok(), + "should not count brackets inside strings" + ); + } + + #[test] + fn raw_json_compares_by_bytes() { + let a = serde_json::value::to_raw_value(&json!({ "k": 1 })).expect("should encode"); + let b = serde_json::value::to_raw_value(&json!({ "k": 1 })).expect("should encode"); + assert_eq!( + RawJson::try_from(a.as_ref()).expect("should accept"), + RawJson::try_from(b.as_ref()).expect("should accept"), + "should compare equal bytes as equal" + ); + } + + #[derive(Debug, PartialEq, Serialize, Deserialize)] + struct WithExpiry { + #[serde( + default, + rename = "exp", + with = "optional_exp", + skip_serializing_if = "Expiry::is_never" + )] + expiry: Expiry, + } + + #[test] + fn optional_exp_omits_never_and_reads_missing() { + let never = serde_json::to_value(WithExpiry { + expiry: Expiry::Never, + }) + .expect("should serialize"); + assert_eq!(never, json!({}), "should omit exp for Never"); + let at: WithExpiry = + serde_json::from_value(json!({ "exp": 42 })).expect("should deserialize exp"); + assert_eq!( + at.expiry, + Expiry::At(UnixSeconds::new(42)), + "should read exp" + ); + let missing: WithExpiry = + serde_json::from_value(json!({})).expect("should deserialize empty"); + assert_eq!(missing.expiry, Expiry::Never, "should default to Never"); + } + + #[test] + fn raw_json_deserialize_applies_same_checks() { + let ok: RawJson = serde_json::from_str(r#"{"k":1}"#).expect("should accept small object"); + assert_eq!(ok.as_str(), r#"{"k":1}"#, "should keep raw bytes"); + assert!( + serde_json::from_str::("[1]").is_err(), + "should reject non-object on deserialize" + ); + let big = serde_json::to_string(&json!({ "k": "x".repeat(MAX_EXT_BYTES) })) + .expect("should encode big object"); + assert!( + serde_json::from_str::(&big).is_err(), + "should reject oversized ext on deserialize" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/eids.rs b/crates/trusted-server-core/src/ec/eids.rs index 1359f75091..9e7a4bad61 100644 --- a/crates/trusted-server-core/src/ec/eids.rs +++ b/crates/trusted-server-core/src/ec/eids.rs @@ -1,102 +1,133 @@ //! Shared EID resolution and formatting helpers. //! -//! Used by both `/_ts/api/v1/identify` and `/auction` to resolve source-domain -//! keyed IDs from KV entries, convert them to `OpenRTB` EID structures, and -//! build base64-encoded response headers. +//! Used by both `/_ts/api/v1/identify` and `/auction` to view source-domain +//! keyed IDs from KV entries as `OpenRTB` EID structures, and to build +//! base64-encoded response headers. use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; use error_stack::{Report, ResultExt as _}; +use crate::consent::ConsentContext; use crate::error::TrustedServerError; use crate::openrtb::{Eid, Uid}; -use super::kv_types::KvEntry; +use super::eid_types::{RawJson, UnixSeconds}; +use super::kv_types::{KvEid, KvEntry, KvPartnerId}; use super::registry::PartnerRegistry; /// Maximum size (in bytes) for the base64-encoded `x-ts-eids` header value. pub const MAX_EIDS_HEADER_BYTES: usize = 4096; -/// A source-domain keyed ID resolved from a KV entry against the partner registry. -/// -/// Only includes partners with `bidstream_enabled = true` and a non-empty UID. -pub struct ResolvedPartnerId { - /// The partner's identity source domain and EC KV `ids` key. - pub source_domain: String, - /// The synced user ID value. - pub uid: String, - /// `OpenRTB` agent type for this partner's identifiers. - pub openrtb_atype: i32, +/// A borrowed EID entry: a single legacy UID or a full-fidelity provider entry. +#[derive(Debug, Clone, Copy)] +pub enum EidView<'row> { + /// A record holding one UID and no EID-level fields. + Single { + /// Source domain. + source: &'row str, + /// The UID. + uid: &'row str, + /// The partner's configured agent type. + atype: i32, + }, + /// A provider-written entry; only live UIDs are emitted. + Full { + /// Source domain. + source: &'row str, + /// The stored entry. + eid: &'row KvEid, + /// Request time used to filter UIDs. + now: UnixSeconds, + }, } -/// Resolves source-domain keyed IDs from a KV entry against the partner registry. -/// -/// Filters to partners with `bidstream_enabled = true` and non-empty UIDs, -/// sorted deterministically by source domain. -#[must_use] -pub fn resolve_partner_ids(registry: &PartnerRegistry, entry: &KvEntry) -> Vec { - let mut resolved = Vec::new(); - - for (source_domain, partner_uid) in &entry.ids { - if partner_uid.uid.is_empty() { - continue; - } - - let Some(partner) = registry.get(source_domain) else { - continue; - }; - if !partner.bidstream_enabled { - continue; +impl From> for Eid { + fn from(view: EidView<'_>) -> Self { + match view { + EidView::Single { source, uid, atype } => Eid::new( + source, + vec![Uid { + id: uid.to_owned(), + atype: Some(atype), + ext: None, + }], + ), + EidView::Full { source, eid, now } => { + let fields = eid.fields(); + let uids = eid + .uids() + .iter() + .filter(|uid| uid.is_live(now)) + .map(|uid| Uid { + id: uid.id().as_str().to_owned(), + atype: Some(uid.atype().get()), + ext: uid.ext().map(RawJson::to_value), + }) + .collect(); + Eid { + inserter: fields.inserter.as_deref().map(str::to_owned), + matcher: fields.matcher.as_deref().map(str::to_owned), + mm: fields.mm.map(i32::from), + ext: fields.ext.as_ref().map(RawJson::to_value), + ..Eid::new(source, uids) + } + } } - - resolved.push(ResolvedPartnerId { - source_domain: partner.source_domain.clone(), - uid: partner_uid.uid.clone(), - openrtb_atype: partner.openrtb_atype, - }); } - - resolved.sort_by(|a, b| a.source_domain.cmp(&b.source_domain)); - resolved } -/// Converts resolved partner IDs to `OpenRTB` `Eid` entries. -#[must_use] -pub fn to_eids(resolved: &[ResolvedPartnerId]) -> Vec { - resolved +/// Yields the live views of one stored record. +/// +/// Legacy records yield one [`EidView::Single`]; provider records yield one +/// [`EidView::Full`] per entry that still has a live UID. +pub fn source_views<'row>( + source: &'row str, + record: &'row KvPartnerId, + default_atype: i32, + now: UnixSeconds, +) -> impl Iterator> + 'row { + let single = + (!record.is_provider_written() && !record.uid.is_empty()).then_some(EidView::Single { + source, + uid: record.uid.as_str(), + atype: default_atype, + }); + let full = record + .eids() .iter() - .map(|item| Eid { - source: item.source_domain.clone(), - uids: vec![Uid { - id: item.uid.clone(), - atype: Some(item.openrtb_atype), - ext: None, - }], - }) - .collect() + .filter(move |eid| eid.uids().iter().any(|uid| uid.is_live(now))) + .map(move |eid| EidView::Full { source, eid, now }); + single.into_iter().chain(full) } -/// Builds a base64-encoded EID header value, truncating if needed. -/// -/// Returns `(encoded_value, was_truncated)`. If the full set of EIDs exceeds -/// [`MAX_EIDS_HEADER_BYTES`] after base64 encoding, partners are removed -/// from the end of the deterministic partner ordering until it fits. -/// -/// # Errors -/// -/// Returns an error if JSON serialization fails. -pub fn build_eids_header( - resolved: &[ResolvedPartnerId], -) -> Result<(String, bool), Report> { - let eids = to_eids(resolved); - encode_eids_header(&eids) +impl KvEntry { + /// Iterates live entries for registered, `bidstream_enabled` sources in + /// source order, borrowing from the row. + /// + /// A source whose owning provider has a TCF vendor ID is skipped while + /// `consent` withholds that vendor's consent. + pub fn live_eids<'row>( + &'row self, + registry: &'row PartnerRegistry, + consent: &'row ConsentContext, + now: UnixSeconds, + ) -> impl Iterator> + 'row { + self.ids.iter().flat_map(move |(source, record)| { + let partner = registry.get(source).filter(|partner| { + partner.bidstream_enabled && !partner.owner_vendor_consent_denied(consent) + }); + partner + .into_iter() + .flat_map(move |partner| source_views(source, record, partner.openrtb_atype, now)) + }) + } } /// Encodes a pre-built EID slice into a base64 header value with truncation. /// -/// Like [`build_eids_header`] but operates on already-constructed `Eid` values -/// (e.g., from `UserInfo.eids` in the auction response path). -/// -/// Returns `(encoded_value, was_truncated)`. +/// Returns `(encoded_value, was_truncated)`. If the full set of EIDs exceeds +/// [`MAX_EIDS_HEADER_BYTES`] after base64 encoding, EIDs are removed from the +/// end until it fits. /// /// # Errors /// @@ -145,8 +176,12 @@ pub fn encode_eids_header(eids: &[Eid]) -> Result<(String, bool), Report EcPartner { EcPartner { @@ -167,113 +202,162 @@ mod tests { } } + fn header_eid(source_domain: String, uid: String) -> Eid { + Eid::new( + source_domain, + vec![Uid { + id: uid, + atype: Some(3), + ext: None, + }], + ) + } + #[test] - fn resolve_partner_ids_sorts_by_source_domain() { - let partners = vec![ - make_test_partner("zeta.example.com"), - make_test_partner("alpha.example.com"), - ]; - let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); + fn encode_eids_header_truncates_when_too_large() { + let eids: Vec = (0..64) + .map(|idx| { + header_eid( + format!("partner-{idx}.example.com"), + format!("uid_{}", "x".repeat(100)), + ) + }) + .collect(); - let mut entry = KvEntry::tombstone(1000); - entry.consent.ok = true; - entry.ids.insert( - "zeta.example.com".to_owned(), - super::super::kv_types::KvPartnerId { - uid: "uid-z".to_owned(), - }, + let (encoded, truncated) = + encode_eids_header(&eids).expect("should build truncated header"); + + assert!(truncated, "should report truncation for large payload"); + assert!( + encoded.len() <= MAX_EIDS_HEADER_BYTES, + "should cap encoded header bytes" ); + } + + #[test] + fn encode_eids_header_fits_without_truncation() { + let eids = vec![header_eid("ssp.example.com".to_owned(), "u1".to_owned())]; + + let (encoded, truncated) = + encode_eids_header(&eids).expect("should build header without truncation"); + + assert!(!truncated, "should not truncate small payload"); + assert!(!encoded.is_empty(), "should produce non-empty value"); + } + + fn registry_with(source: &str, bidstream: bool) -> PartnerRegistry { + let mut partner = make_test_partner(source); + partner.bidstream_enabled = bidstream; + PartnerRegistry::from_config(&[partner]).expect("should build registry") + } + + fn provider_record(ids: &[(&str, Expiry)]) -> KvPartnerId { + let uids = ids + .iter() + .map(|(id, expiry)| { + KvUid::new( + UidValue::try_from(*id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + *expiry, + ) + }) + .collect(); + let fields = EidFields { + inserter: Some("inserter.example.com".into()), + ..EidFields::default() + }; + KvPartnerId::from_eids(vec![KvEid::new(fields, uids)], UnixSeconds::new(0)) + .expect("should build record") + } + + #[test] + fn live_eids_skip_expired_uids_and_keep_eid_fields() { + let registry = registry_with("ids.example.com", true); + let mut entry = KvEntry::minimal("ids.example.com", "placeholder", 1); entry.ids.insert( - "alpha.example.com".to_owned(), - super::super::kv_types::KvPartnerId { - uid: "uid-a".to_owned(), - }, + "ids.example.com".to_owned(), + provider_record(&[ + ("old", Expiry::At(UnixSeconds::new(10))), + ("new", Expiry::Never), + ]), ); - let resolved = resolve_partner_ids(®istry, &entry); - let source_domains: Vec<&str> = resolved - .iter() - .map(|item| item.source_domain.as_str()) + let eids: Vec = entry + .live_eids(®istry, &ConsentContext::default(), UnixSeconds::new(20)) + .map(Eid::from) .collect(); + assert_eq!(eids.len(), 1, "should emit one entry"); assert_eq!( - source_domains, - vec!["alpha.example.com", "zeta.example.com"], - "should sort deterministically by source domain" + eids[0].inserter.as_deref(), + Some("inserter.example.com"), + "should keep inserter" ); + let ids: Vec<&str> = eids[0].uids.iter().map(|uid| uid.id.as_str()).collect(); + assert_eq!(ids, vec!["new"], "should drop expired uid"); } #[test] - fn to_eids_maps_resolved_ids_correctly() { - let resolved = vec![ - ResolvedPartnerId { - uid: "LR_xyz".to_owned(), - source_domain: "liveramp.com".to_owned(), - openrtb_atype: 3, - }, - ResolvedPartnerId { - uid: "ID5_abc".to_owned(), - source_domain: "id5-sync.com".to_owned(), - openrtb_atype: 1, - }, - ResolvedPartnerId { - uid: "pair-id".to_owned(), - source_domain: "google.com".to_owned(), - openrtb_atype: 571187, - }, - ]; - - let eids = to_eids(&resolved); + fn live_eids_withhold_a_source_whose_owner_lacks_vendor_consent() { + let mut registry = registry_with("ids.example.com", true); + assert!( + registry.mark_provider_owned("ids.example.com", NonZeroU16::new(9999)), + "should mark the source" + ); + let mut entry = KvEntry::minimal("ids.example.com", "placeholder", 1); + entry.ids.insert( + "ids.example.com".to_owned(), + provider_record(&[("p-1", Expiry::Never)]), + ); + let denied = consent_granting_vendors(vec![1]); + let granted = consent_granting_vendors(vec![9999]); - assert_eq!(eids.len(), 3, "should produce one EID per resolved partner"); - assert_eq!(eids[0].source, "liveramp.com"); - assert_eq!(eids[0].uids[0].id, "LR_xyz"); - assert_eq!(eids[0].uids[0].atype, Some(3)); - assert_eq!(eids[1].source, "id5-sync.com"); - assert_eq!(eids[1].uids[0].id, "ID5_abc"); - assert_eq!(eids[1].uids[0].atype, Some(1)); - assert_eq!(eids[2].source, "google.com", "should preserve PAIR source"); - assert_eq!(eids[2].uids[0].id, "pair-id", "should preserve PAIR ID"); assert_eq!( - eids[2].uids[0].atype, - Some(571187), - "should preserve PAIR vendor-specific atype" + entry + .live_eids(®istry, &denied, UnixSeconds::new(0)) + .count(), + 0, + "should withhold the provider's UIDs without its vendor consent" + ); + assert_eq!( + entry + .live_eids(®istry, &granted, UnixSeconds::new(0)) + .count(), + 1, + "should serve the provider's UIDs with its vendor consent" ); } #[test] - fn build_eids_header_truncates_when_too_large() { - let mut resolved = Vec::new(); - for idx in 0..64 { - resolved.push(ResolvedPartnerId { - uid: format!("uid_{}", "x".repeat(100)), - source_domain: format!("partner-{idx}.example.com"), - openrtb_atype: 3, - }); - } - - let (encoded, truncated) = - build_eids_header(&resolved).expect("should build truncated header"); - - assert!(truncated, "should report truncation for large payload"); - assert!( - encoded.len() <= MAX_EIDS_HEADER_BYTES, - "should cap encoded header bytes" + fn live_eids_respect_bidstream_flag() { + let registry = registry_with("ids.example.com", false); + let mut entry = KvEntry::minimal("ids.example.com", "v", 1); + entry + .ids + .insert("ids.example.com".to_owned(), KvPartnerId::single("v")); + assert_eq!( + entry + .live_eids(®istry, &ConsentContext::default(), UnixSeconds::new(0)) + .count(), + 0, + "should hide non-bidstream source" ); } #[test] - fn build_eids_header_fits_without_truncation() { - let resolved = vec![ResolvedPartnerId { - uid: "u1".to_owned(), - source_domain: "ssp.com".to_owned(), - openrtb_atype: 3, - }]; - - let (encoded, truncated) = - build_eids_header(&resolved).expect("should build header without truncation"); - - assert!(!truncated, "should not truncate small payload"); - assert!(!encoded.is_empty(), "should produce non-empty value"); + fn live_eids_emit_legacy_single_uid() { + let registry = registry_with("ids.example.com", true); + let mut entry = KvEntry::minimal("ids.example.com", "v", 1); + entry + .ids + .insert("ids.example.com".to_owned(), KvPartnerId::single("v")); + let eids: Vec = entry + .live_eids(®istry, &ConsentContext::default(), UnixSeconds::new(0)) + .map(Eid::from) + .collect(); + assert_eq!(eids[0].uids[0].id, "v", "should emit legacy uid"); + assert_eq!(eids[0].uids[0].atype, Some(3), "should use partner atype"); } } diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 019ef2aa40..b385414902 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -1,9 +1,12 @@ //! EC response finalization. //! //! Centralizes post-routing EC behavior so all handlers get consistent cookie -//! and KV semantics. +//! and KV semantics. Every browser-driven KV write goes through the +//! navigation's single page-load write ([`KvIdentityGraph::commit_page_load`]), +//! so a request without a navigation permit never writes. The one exception is +//! a consent withdrawal, whose tombstone is written on any request. -use std::collections::HashSet; +use std::collections::BTreeSet; use edgezero_core::body::Body as EdgeBody; use http::Response; @@ -14,16 +17,16 @@ use crate::settings::Settings; use super::EcContext; use super::cookies::{expire_ec_cookie, set_ec_cookie}; +use super::eid_types::UnixSeconds; use super::generation::{generate_ec_id, is_valid_ec_id}; -use super::kv::{ - CreateIfAbsentOutcome, EidCookieSyncOutcome, KvIdentityGraph, PartnerIdUpdate, - apply_partner_id_updates, -}; +use super::kv::{KvIdentityGraph, PartnerIdUpdate}; use super::kv_types::KvEntry; +use super::page_load::{PageLoadWrite, RowChange}; use super::prebid_eids::collect_eid_updates; use super::pull_sync_marker::{expire_marker, reconcile_marker}; use super::registry::PartnerRegistry; -use super::{EcKvSnapshot, EidSyncSource, current_timestamp, log_id}; +use super::write_plan::{WritePlanInput, plan_write}; +use super::{EcKvSnapshot, current_timestamp, log_id}; /// TS-managed response headers tied to EC identity output. const EC_RESPONSE_HEADERS: &[&str] = &[ @@ -35,12 +38,18 @@ const EC_RESPONSE_HEADERS: &[&str] = &[ /// Finalizes EC response behavior for all routes. /// -/// Applies withdrawal handling, last-seen updates, cookie reconciliation, -/// Prebid EID ingestion, and cookie writes for new EC generation. +/// Applies withdrawal handling, browser EID ingestion, provider lookup +/// results, orphan recovery and cookie writes for new EC generation. /// -/// On consent withdrawal, the browser response clears the EC cookie -/// immediately and the EC identity-graph KV tombstone is the authoritative -/// revocation marker. There is no separate consent KV store to clean up. +/// The identity row is written at most once, and only when the request +/// carries a navigation permit (see [`super::page_load`]) and the final +/// response is not a redirect. Without a permit a newly generated EC is not +/// issued. +/// +/// Consent withdrawal is the one exception: on any request carrying the EC +/// cookie, the KV tombstone is written as the authoritative revocation marker +/// and the EC cookie is expired once every tombstone is in place (or the row +/// is proven absent). /// /// `eids_cookie` should be the raw value of the `ts-eids` cookie extracted /// from the request *before* routing consumes it. @@ -54,6 +63,10 @@ pub fn ec_finalize_response( response: &mut Response, ) { ec_context.validate_pull_sync_marker(settings, registry); + let (permit, after_response) = ec_context + .take_navigation_permit() + .and_then(|permit| permit.confirm_with_after_response(response.status())) + .unzip(); let consent_allows_ec = ec_consent_granted(ec_context.consent()); let consent_withdrawn = ec_consent_withdrawn(ec_context.consent()); @@ -75,153 +88,197 @@ pub fn ec_finalize_response( return; } - // Returning user: consent is granted and EC came from request. - if ec_context.ec_was_present() && !ec_context.ec_generated() && consent_allows_ec { - if let (Some(graph), Some(ec_id)) = (kv, ec_context.ec_value().map(str::to_owned)) { - let source = ec_context.eid_sync_source(); - let collected = source - .map(|_| { - collect_eid_updates( - eids_cookie, - sharedid_cookie, - ec_context.client_eids(), - registry, - ) - }) - .unwrap_or_default(); - let had_eid_updates = !collected.is_empty(); - let updates = gate_eid_updates_by_consent(collected, ec_context.consent()); - // `sync_eid_cookie_updates` early-returns without refreshing the - // snapshot when `updates` is empty, which is correct when there - // was never anything to write (an unconfigured registry, no sync - // source, or no EID cookies/body this request — the case - // `finalize_not_read_snapshot_does_not_rotate` covers). But when - // consent gating (e.g. TCF Purpose 4 denial) is what emptied - // `updates`, the request *did* have EID data, and orphan recovery - // below still needs an actual `Missing` read to detect an - // orphaned cookie. An unread `NotRead` snapshot proves nothing, so - // consent gating must not also skip that one read on a - // recovery-eligible request. - if updates.is_empty() - && had_eid_updates - && ec_context.recovery_eligible() - && matches!(ec_context.kv_snapshot(), EcKvSnapshot::NotRead) - { - let snapshot = graph.load_snapshot(&ec_id); - ec_context.set_kv_snapshot(snapshot); - } else if let Some(source) = source { - sync_eid_cookie_updates(graph, ec_context, &ec_id, &updates, source); - } - if matches!(ec_context.kv_snapshot(), EcKvSnapshot::Missing { .. }) - && ec_context.recovery_eligible() - { - confirm_then_recover_orphaned_ec( - settings, ec_context, graph, &ec_id, &updates, response, - ); - } - } - - reconcile_pull_sync_marker(settings, registry, ec_context, response); - - // Ordinary returning-user page views no longer refresh the browser - // cookie, emit the EC header, or update KV TTL. - return; - } - - // Newly generated EC in this request. Do not emit a generated EC when - // there is no KV graph: that would mint a browser cookie with no backing - // identity-graph row, producing a phantom ID on later requests. - if ec_context.ec_generated() { - let (Some(graph), Some(ec_id)) = (kv, ec_context.ec_value().map(str::to_owned)) else { - log::info!("Skipping generated EC response write because KV graph is unavailable"); - reconcile_pull_sync_marker(settings, registry, ec_context, response); - return; - }; - - let updates = gate_eid_updates_by_consent( + if let (Some(graph), Some(ec_id)) = (kv, ec_context.ec_value().map(str::to_owned)) { + let collected = if ec_context.eid_sync_source().is_some() || ec_context.ec_generated() { collect_eid_updates( eids_cookie, sharedid_cookie, ec_context.client_eids(), registry, - ), - ec_context.consent(), - ); - sync_eid_cookie_updates(graph, ec_context, &ec_id, &updates, EidSyncSource::NewEc); - if ec_context.kv_snapshot().entry_for(&ec_id).is_some() { - set_ec_cookie_on_response(settings, ec_context, response); + ) } else { - log::warn!("Skipping generated EC cookie because backing row is not authoritative"); + Vec::new() + }; + let had_eid_updates = !collected.is_empty(); + let updates = gate_eid_updates_by_consent(collected, ec_context.consent()); + let commit = PageLoadCommit { + settings, + graph, + registry, + ec_id: &ec_id, + updates: &updates, + had_eid_updates, + }; + let page_write_ok = if ec_context.ec_generated() { + commit_generated(&commit, ec_context, permit, response) + } else if ec_context.ec_was_present() { + commit_returning(&commit, ec_context, permit, response) + } else { + false + }; + // The follow-up write is allowed only once consent allows the EC and + // the page-load write succeeded or had nothing to change. + if page_write_ok && let Some(after_response) = after_response { + ec_context.set_after_response_permit(after_response); } + } else if ec_context.ec_generated() { + // Do not emit a generated EC when there is no KV graph: that would + // mint a browser cookie with no backing identity-graph row. + log::info!("Skipping generated EC response write because KV graph is unavailable"); } reconcile_pull_sync_marker(settings, registry, ec_context, response); } -fn sync_eid_cookie_updates( - graph: &KvIdentityGraph, - ec_context: &mut EcContext, - ec_id: &str, - updates: &[PartnerIdUpdate], - source: EidSyncSource, -) { - if updates.is_empty() { - return; - } +/// Shared inputs for the navigation's single write. +struct PageLoadCommit<'a> { + settings: &'a Settings, + graph: &'a KvIdentityGraph, + registry: &'a PartnerRegistry, + ec_id: &'a str, + /// Browser ingest, already consent-gated. + updates: &'a [PartnerIdUpdate], + /// Whether the request carried EID data before consent gating. + had_eid_updates: bool, +} - let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - ec_id, - updates, - ec_context.kv_snapshot().clone(), - ); - ec_context.set_kv_snapshot(snapshot); - record_eid_sync_terminal(source, outcome); +impl PageLoadCommit<'_> { + /// Plans the row change for `ec_id` against `snapshot`. + fn plan( + &self, + ec_context: &EcContext, + ec_id: &str, + snapshot: &EcKvSnapshot, + new_entry: Option<&KvEntry>, + ) -> Option { + plan_write(&WritePlanInput { + ec_id, + snapshot, + new_entry, + browser_ingest: self.updates, + lookups: ec_context.lookup_results(), + registry: self.registry, + now: UnixSeconds::new(current_timestamp()), + }) + } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct EidSyncMeasurement { - source: EidSyncSource, - outcome: EidCookieSyncOutcome, - already_matched: u8, - written: u8, - conflict_duplicate: u8, - deferred: u8, +fn present_snapshot(ec_id: &str, entry: KvEntry) -> EcKvSnapshot { + EcKvSnapshot::Present { + ec_id: ec_id.to_owned(), + entry: Box::new(entry), + generation: None, + } } -impl EidSyncMeasurement { - fn new(source: EidSyncSource, outcome: EidCookieSyncOutcome) -> Self { - Self { - source, - outcome, - already_matched: u8::from(matches!(outcome, EidCookieSyncOutcome::AlreadyMatched)), - written: u8::from(matches!( - outcome, - EidCookieSyncOutcome::Written | EidCookieSyncOutcome::WrittenWithDeferredFreshness - )), - conflict_duplicate: u8::from(matches!(outcome, EidCookieSyncOutcome::ConflictMatched)), - deferred: u8::from(matches!( - outcome, - EidCookieSyncOutcome::WrittenWithDeferredFreshness - | EidCookieSyncOutcome::DeferredConflict - | EidCookieSyncOutcome::DeferredFreshness - | EidCookieSyncOutcome::DeferredStaleRead - )), +/// Creates the row of a newly generated EC and issues its cookie on success. +/// +/// Returns whether the row was created. +fn commit_generated( + commit: &PageLoadCommit<'_>, + ec_context: &mut EcContext, + permit: Option, + response: &mut Response, +) -> bool { + let Some(new_entry) = ec_context.take_pending_entry() else { + log::warn!("Skipping generated EC cookie because no pending row was built"); + return false; + }; + let Some(permit) = permit else { + log::info!("Skipping generated EC: not a page-load navigation"); + return false; + }; + let Some(change) = commit.plan( + ec_context, + commit.ec_id, + ec_context.kv_snapshot(), + Some(&new_entry), + ) else { + log::warn!("Skipping generated EC: no row change was planned"); + return false; + }; + match commit.graph.commit_page_load(permit, commit.ec_id, change) { + Ok(committed) => { + ec_context.set_kv_snapshot(present_snapshot(commit.ec_id, committed)); + set_ec_cookie_on_response(commit.settings, ec_context, response); + true + } + Err(report) => { + log::warn!( + "Page-load write failed for new EC '{}': {}", + log_id(commit.ec_id), + report.current_context() + ); + false } } } -fn record_eid_sync_terminal(source: EidSyncSource, outcome: EidCookieSyncOutcome) { - let measurement = EidSyncMeasurement::new(source, outcome); - log::info!( - "EID sync measurement: source={} outcome={} attempted=1 already_matched={} written={} \ - conflict_duplicate={} deferred={}", - measurement.source, - measurement.outcome, - measurement.already_matched, - measurement.written, - measurement.conflict_duplicate, - measurement.deferred, - ); +/// Applies a returning EC's page-load write, or recovers an orphaned EC. +/// +/// Returns whether the row is live afterwards: written, recovered, or left +/// unchanged because there was nothing to write. +fn commit_returning( + commit: &PageLoadCommit<'_>, + ec_context: &mut EcContext, + permit: Option, + response: &mut Response, +) -> bool { + let Some(permit) = permit else { + return false; + }; + // An unread snapshot is read only when this navigation has something to + // write, or when consent gating emptied real EID data on a request that + // may still need orphan recovery. A navigation with nothing to do costs + // no KV operation. + let has_changes = !commit.updates.is_empty() || !ec_context.lookup_results().is_empty(); + if matches!(ec_context.kv_snapshot(), EcKvSnapshot::NotRead) + && (has_changes || (commit.had_eid_updates && ec_context.recovery_eligible())) + { + let snapshot = commit.graph.load_snapshot(commit.ec_id); + ec_context.set_kv_snapshot(snapshot); + } + if matches!(ec_context.kv_snapshot(), EcKvSnapshot::Missing { .. }) { + return ec_context.recovery_eligible() + && confirm_then_recover_orphaned_ec(commit, ec_context, permit, response); + } + if matches!(ec_context.kv_snapshot(), EcKvSnapshot::Failed { .. }) { + return false; + } + update_returning(commit, ec_context, permit) +} + +/// Writes the planned update for a returning EC whose row snapshot is held. +/// +/// Returns `false` when a planned write failed, or when there was nothing to +/// write because the row is withdrawn. +fn update_returning( + commit: &PageLoadCommit<'_>, + ec_context: &mut EcContext, + permit: PageLoadWrite, +) -> bool { + let Some(change) = commit.plan(ec_context, commit.ec_id, ec_context.kv_snapshot(), None) else { + // Nothing to write. A withdrawn row is not live and must not unlock + // the follow-up write. + return ec_context + .kv_snapshot() + .entry_for(commit.ec_id) + .is_none_or(|entry| entry.consent.ok); + }; + match commit.graph.commit_page_load(permit, commit.ec_id, change) { + Ok(committed) => { + ec_context.set_kv_snapshot(present_snapshot(commit.ec_id, committed)); + true + } + Err(report) => { + log::info!( + "Page-load write skipped for '{}': {}", + log_id(commit.ec_id), + report.current_context() + ); + false + } + } } fn reconcile_pull_sync_marker( @@ -252,12 +309,11 @@ fn reconcile_pull_sync_marker( /// the outbound `/auction` bid request, evaluated here through /// [`allows_eid_persistence`]. Without this, a user who denies Purpose 4 /// would have their EIDs correctly stripped from the bid request but still -/// written to the identity graph from the `ts-eids`/`sharedId` cookies or the -/// `/auction` request body. +/// written to the identity graph from the `ts-eids`/`sharedId` cookies. fn gate_eid_updates_by_consent( - updates: Vec, + updates: Vec, consent: &ConsentContext, -) -> Vec { +) -> Vec { if updates.is_empty() || allows_eid_persistence(consent) { return updates; } @@ -268,75 +324,60 @@ fn gate_eid_updates_by_consent( Vec::new() } +/// Replaces a proven-orphaned EC with a new ID whose row is created by this +/// navigation's single write. One attempt: a collision leaves the orphan in +/// place for a later navigation. fn recover_orphaned_ec( - settings: &Settings, + commit: &PageLoadCommit<'_>, ec_context: &mut EcContext, - graph: &KvIdentityGraph, - updates: &[super::kv::PartnerIdUpdate], + permit: PageLoadWrite, response: &mut Response, ) { - // Snapshot the orphaned ID once so every fail-closed exit binds the failed - // snapshot to the same key. - let Some(orphan_id) = ec_context.ec_value().map(str::to_owned) else { - return; - }; + let orphan_id = commit.ec_id.to_owned(); let Some(client_ip) = ec_context.client_ip().map(str::to_owned) else { log::warn!("Orphan EC recovery skipped because client IP is unavailable"); - ec_context.set_kv_snapshot(EcKvSnapshot::Failed { - ec_id: orphan_id.clone(), - }); + ec_context.set_kv_snapshot(EcKvSnapshot::Failed { ec_id: orphan_id }); return; }; + let new_id = match generate_ec_id(commit.settings, &client_ip) { + Ok(ec_id) => ec_id, + Err(err) => { + log::warn!("Orphan EC recovery ID generation failed: {err:?}"); + ec_context.set_kv_snapshot(EcKvSnapshot::Failed { ec_id: orphan_id }); + return; + } + }; + let mut new_entry = KvEntry::new( + ec_context.consent(), + ec_context.geo_info(), + current_timestamp(), + &commit.settings.publisher.domain, + ); + new_entry.device = ec_context + .device_signals() + .map(super::device::DeviceSignals::to_kv_device); - const MAX_RECOVERY_ATTEMPTS: usize = 5; - for _attempt in 0..MAX_RECOVERY_ATTEMPTS { - let ec_id = match generate_ec_id(settings, &client_ip) { - Ok(ec_id) => ec_id, - Err(err) => { - log::warn!("Orphan EC recovery ID generation failed: {err:?}"); - ec_context.set_kv_snapshot(EcKvSnapshot::Failed { - ec_id: orphan_id.clone(), - }); - return; - } - }; - let mut entry = KvEntry::new( - ec_context.consent(), - ec_context.geo_info(), - current_timestamp(), - &settings.publisher.domain, - ); - entry.device = ec_context - .device_signals() - .map(super::device::DeviceSignals::to_kv_device); - apply_partner_id_updates(&mut entry, updates); - - match graph.create_if_absent(&ec_id, &entry) { - Ok(CreateIfAbsentOutcome::Written) => { - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(entry), - generation: None, - }; - ec_context.replace_with_generated(ec_id, snapshot); - set_ec_cookie_on_response(settings, ec_context, response); - return; - } - Ok(CreateIfAbsentOutcome::AlreadyExists) => continue, - Err(err) => { - log::warn!("Orphan EC recovery failed: {err:?}"); - ec_context.set_kv_snapshot(EcKvSnapshot::Failed { - ec_id: orphan_id.clone(), - }); - return; - } + let missing = EcKvSnapshot::Missing { + ec_id: new_id.clone(), + }; + let Some(change) = commit.plan(ec_context, &new_id, &missing, Some(&new_entry)) else { + log::warn!("Orphan EC recovery skipped: no row change was planned"); + return; + }; + match commit.graph.commit_page_load(permit, &new_id, change) { + Ok(committed) => { + let snapshot = present_snapshot(&new_id, committed); + ec_context.replace_with_generated(new_id, snapshot); + set_ec_cookie_on_response(commit.settings, ec_context, response); + } + Err(report) => { + log::warn!( + "Orphan EC recovery write failed: {}", + report.current_context() + ); + ec_context.set_kv_snapshot(EcKvSnapshot::Failed { ec_id: orphan_id }); } } - - log::warn!("Orphan EC recovery exhausted collision retries"); - ec_context.set_kv_snapshot(EcKvSnapshot::Failed { - ec_id: orphan_id.clone(), - }); } /// Proves an orphaned cookie is genuinely absent before rotating it. @@ -348,8 +389,8 @@ fn recover_orphaned_ec( /// POP that has not converged. Two point reads do not fix that — both can be /// stale — so absence has to be *proved*, not observed twice: /// -/// - a row that became visible after the origin round trip is adopted, with any -/// pending updates merged, and is never rotated; +/// - a row that became visible after the origin round trip is adopted, and the +/// navigation's single write is planned against it; it is never rotated; /// - a second miss is escalated to /// [`key_exists_confirmed`](KvIdentityGraph::key_exists_confirmed), which /// reads the primary data source. Only a proven-absent key rotates; @@ -357,37 +398,44 @@ fn recover_orphaned_ec( /// the identity stays intact and recovery is retried on a later navigation; /// - neither a read failure nor a failed existence check is a miss, and neither /// rotates. +/// +/// Returns whether the navigation's write succeeded or had nothing to change, +/// which is what allows the after-response follow-up write. fn confirm_then_recover_orphaned_ec( - settings: &Settings, + commit: &PageLoadCommit<'_>, ec_context: &mut EcContext, - graph: &KvIdentityGraph, - ec_id: &str, - updates: &[super::kv::PartnerIdUpdate], + permit: PageLoadWrite, response: &mut Response, -) { - let confirmed = graph.load_snapshot(ec_id); +) -> bool { + let confirmed = commit.graph.load_snapshot(commit.ec_id); match confirmed { EcKvSnapshot::Present { .. } => { // The row became visible after the origin round trip: adopt it and - // merge any pending updates rather than rotating a valid identity. + // plan the navigation's write against it rather than rotating a + // valid identity. ec_context.set_kv_snapshot(confirmed); - sync_eid_cookie_updates(graph, ec_context, ec_id, updates, EidSyncSource::Navigation); + update_returning(commit, ec_context, permit) } - EcKvSnapshot::Missing { .. } => match graph.key_exists_confirmed(ec_id) { - Ok(false) => recover_orphaned_ec(settings, ec_context, graph, updates, response), + EcKvSnapshot::Missing { .. } => match commit.graph.key_exists_confirmed(commit.ec_id) { + Ok(false) => { + recover_orphaned_ec(commit, ec_context, permit, response); + matches!(ec_context.kv_snapshot(), EcKvSnapshot::Present { .. }) + } Ok(true) => { log::warn!( "Orphan EC recovery skipped: both point reads missed a row the store still \ lists; leaving the identity intact for a later navigation" ); + false } Err(err) => { log::warn!("Orphan EC recovery skipped: existence check failed: {err:?}"); + false } }, // A failed or not-read confirmation is not an authoritative miss: leave // the existing snapshot in place and do not rotate an unconfirmed miss. - EcKvSnapshot::Failed { .. } | EcKvSnapshot::NotRead => {} + EcKvSnapshot::Failed { .. } | EcKvSnapshot::NotRead => false, } } @@ -439,10 +487,17 @@ pub fn clear_ec_on_response(settings: &Settings, response: &mut Response HashSet { - let mut hashes = HashSet::new(); - - if let Some(cookie_ec_id) = ec_context.existing_cookie_ec_id() - && is_valid_ec_id(cookie_ec_id) - { - hashes.insert(cookie_ec_id.to_owned()); + let mut complete = true; + for ec_id in withdrawal_ec_ids(ec_context) { + // Only the active ID has a snapshot in the context to start from and + // to correct, so post-send work sees the tombstone just written. + let initial = if ec_context.kv_snapshot().belongs_to(&ec_id) { + ec_context.kv_snapshot().clone() + } else { + EcKvSnapshot::NotRead + }; + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, initial); + if matches!(outcome, EcKvSnapshot::Failed { .. }) { + log::warn!( + "EC withdrawal tombstone for '{}' failed; keeping the cookie so the next \ + request retries", + log_id(&ec_id) + ); + complete = false; + } + if ec_context.ec_value() == Some(ec_id.as_str()) { + ec_context.set_kv_snapshot(outcome); + } } - - if let Some(active_ec_id) = ec_context.ec_value() - && is_valid_ec_id(active_ec_id) - { - hashes.insert(active_ec_id.to_owned()); + if complete { + expire_ec_cookie(settings, response); } - - hashes } -fn apply_withdrawal_tombstones(ec_ids: &HashSet, mut write_tombstone: F) -where - F: FnMut(&str), -{ - for ec_id in ec_ids { - write_tombstone(ec_id); - } +/// The EC IDs a withdrawal applies to: the cookie's and the active one. +fn withdrawal_ec_ids(ec_context: &EcContext) -> BTreeSet { + [ec_context.existing_cookie_ec_id(), ec_context.ec_value()] + .into_iter() + .flatten() + .filter(|ec_id| is_valid_ec_id(ec_id)) + .map(str::to_owned) + .collect() } #[cfg(test)] @@ -531,6 +564,13 @@ mod tests { use super::*; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ConsentContext, ConsentSource}; + use crate::ec::EidSyncSource; + use crate::ec::coordinator::results::{ + DropCounts, LookupResults, LookupStatus, ProviderLookup, + }; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, SourceDomain, UidValue}; + use crate::ec::kv_types::{KvEid, KvUid}; + use crate::ec::page_load::NavigationPermit; use crate::openrtb::{Eid, Uid}; use crate::redacted::Redacted; use crate::settings::EcPartner; @@ -620,90 +660,6 @@ mod tests { } } - #[test] - fn withdrawal_ec_ids_returns_cookie_ec_only_when_active_missing() { - let cookie_ec = sample_ec_id("cook1e"); - let ec_context = make_context(None, Some(&cookie_ec), true, false, Jurisdiction::Unknown); - - let ids = withdrawal_ec_ids(&ec_context); - - assert_eq!(ids.len(), 1, "should include exactly one EC ID"); - assert!( - ids.contains(&cookie_ec), - "should include the cookie EC value" - ); - } - - #[test] - fn withdrawal_ec_ids_deduplicates_matching_cookie_and_active_ec() { - let ec_id = sample_ec_id("same01"); - let ec_context = make_context( - Some(&ec_id), - Some(&ec_id), - true, - false, - Jurisdiction::Unknown, - ); - - let ids = withdrawal_ec_ids(&ec_context); - - assert_eq!(ids.len(), 1, "should deduplicate identical EC IDs"); - assert!(ids.contains(&ec_id), "should retain the shared EC ID"); - } - - #[test] - fn withdrawal_ec_ids_includes_both_cookie_and_active_when_different() { - let active_ec = sample_ec_id("activ1"); - let cookie_ec = sample_ec_id("cook1e"); - let ec_context = make_context( - Some(&active_ec), - Some(&cookie_ec), - true, - false, - Jurisdiction::Unknown, - ); - - let ids = withdrawal_ec_ids(&ec_context); - - assert_eq!(ids.len(), 2, "should include both distinct EC IDs"); - assert!(ids.contains(&active_ec), "should include active EC ID"); - assert!(ids.contains(&cookie_ec), "should include cookie EC ID"); - } - - #[test] - fn withdrawal_ec_ids_filters_invalid_values() { - let valid_ec = sample_ec_id("valid1"); - let ec_context = make_context( - Some(&valid_ec), - Some("not-an-ec-id"), - true, - false, - Jurisdiction::Unknown, - ); - - let ids = withdrawal_ec_ids(&ec_context); - - assert_eq!(ids.len(), 1, "should ignore malformed EC values"); - assert!(ids.contains(&valid_ec), "should keep the valid EC ID"); - } - - #[test] - fn apply_withdrawal_tombstones_invokes_writer_for_each_ec_id() { - let first = sample_ec_id("first1"); - let second = sample_ec_id("second"); - let mut ids = HashSet::new(); - ids.insert(first.clone()); - ids.insert(second.clone()); - - let mut written = Vec::new(); - apply_withdrawal_tombstones(&ids, |ec_id| written.push(ec_id.to_owned())); - written.sort(); - - let mut expected = vec![first, second]; - expected.sort(); - assert_eq!(written, expected, "should write a tombstone for each EC ID"); - } - #[test] fn clear_ec_on_response_removes_headers_and_expires_cookie() { let settings = create_test_settings(); @@ -755,6 +711,7 @@ mod tests { let mut ec_context = make_context_with_consent(Some(&ec_id), Some(&ec_id), true, false, consent); let kv = KvIdentityGraph::in_memory("test-store"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); let registry = PartnerRegistry::from_config(&[]).expect("should build registry"); @@ -807,6 +764,7 @@ mod tests { ec_context.kv_snapshot(), EcKvSnapshot::Missing { .. } )); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); let registry = PartnerRegistry::from_config(&[]).expect("should build registry"); @@ -834,13 +792,17 @@ mod tests { .expect("should replace the stale miss with a tombstone snapshot"); assert!(!snapshot_entry.consent.ok && snapshot_entry.ids.is_empty()); assert_eq!(ec_context.kv_snapshot().generation_for(&ec_id), None); + assert!( + ec_context.take_after_response_permit().is_none(), + "should not allow after-response lookups after a withdrawal" + ); } #[test] - fn withdrawal_still_expires_the_cookie_when_the_store_is_unavailable() { - // Cookie expiry is the primary enforcement, so it has to survive a - // store that cannot answer at all — the case where the identity-graph - // marker is exactly what goes missing. + fn withdrawal_keeps_the_cookie_when_the_tombstone_write_fails() { + // A tombstone write that still fails after its bounded CAS retries + // keeps the EC cookie, so the next request that carries it retries; + // clearing it would orphan a live row (spec §9.2). let settings = create_test_settings(); let ec_id = sample_ec_id("dead01"); let consent = ConsentContext { @@ -856,6 +818,7 @@ mod tests { entry: Box::new(live_entry()), generation: Some(1), }); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let kv = KvIdentityGraph::failing("test-store"); let mut response = empty_response(); set_header(&mut response, "x-ts-ec", "stale"); @@ -871,10 +834,9 @@ mod tests { &mut response, ); - let set_cookie = get_header_str(&response, "set-cookie").unwrap_or_default(); assert!( - set_cookie.contains("Max-Age=0"), - "should expire the EC cookie even when the store is unavailable: {set_cookie}" + get_header(&response, "set-cookie").is_none(), + "should keep the EC cookie when the tombstone write fails" ); assert!( get_header(&response, "x-ts-ec").is_none(), @@ -898,6 +860,7 @@ mod tests { }; let mut ec_context = make_context_with_consent(Some(&ec_id), Some(&ec_id), true, false, consent); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); set_header(&mut response, "x-ts-ec", "stale"); set_header(&mut response, "x-ts-eids", "[]"); @@ -1065,6 +1028,7 @@ mod tests { ec_id: orphaned_ec.clone(), }); let graph = KvIdentityGraph::in_memory("test_store"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); ec_finalize_response( @@ -1115,17 +1079,18 @@ mod tests { ); ec_context.set_recovery_eligible(true); ec_context.set_eid_sync_source(EidSyncSource::Navigation); - ec_context.set_client_eids(vec![Eid { - source: "id5-sync.com".to_owned(), - uids: vec![Uid { + ec_context.set_client_eids(vec![Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { id: "ID5_should_not_persist".to_owned(), atype: Some(1), ext: None, }], - }]); + )]); let partners = vec![make_partner("id5-sync.com")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); let graph = KvIdentityGraph::in_memory("test_store"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); ec_finalize_response( @@ -1181,6 +1146,7 @@ mod tests { partner.ts_pull_token = Some(Redacted::new("pull-token".to_owned())); let registry = PartnerRegistry::from_config(&[partner]).expect("should build registry"); let graph = KvIdentityGraph::in_memory("test_store"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); ec_finalize_response( @@ -1240,7 +1206,7 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should remain"); + .expect("should keep the row"); assert!( !stored.ids.contains_key("sharedid.org"), "a subresource response must not persist request EID cookies" @@ -1248,10 +1214,22 @@ mod tests { } #[test] - fn finalize_navigation_routes_persist_returning_user_eid_updates() { - for (source, suffix, cookie_id) in [ - (EidSyncSource::Navigation, "naveid", "navigation-cookie-id"), - (EidSyncSource::PageBids, "spaeid", "page-bids-cookie-id"), + fn finalize_persists_returning_user_eid_updates_only_on_navigation() { + // `/_ts/page-bids` belongs to a page load whose navigation already + // made the write, so it never receives a permit (spec D1). + for (source, suffix, cookie_id, permit) in [ + ( + EidSyncSource::Navigation, + "naveid", + "navigation-cookie-id", + true, + ), + ( + EidSyncSource::PageBids, + "spaeid", + "page-bids-cookie-id", + false, + ), ] { let settings = create_test_settings(); let ec_id = sample_ec_id(suffix); @@ -1265,7 +1243,9 @@ mod tests { graph .create(&ec_id, &live) .expect("should seed the live row"); - let mut ec_context = returning_user_context(&ec_id, graph.load_snapshot(&ec_id), true); + // A recovery-eligible context is a navigation and carries a permit. + let mut ec_context = + returning_user_context(&ec_id, graph.load_snapshot(&ec_id), permit); ec_context.set_eid_sync_source(source); let partners = vec![make_partner("sharedid.org")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); @@ -1284,11 +1264,11 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should remain"); + .expect("should keep the row"); assert_eq!( stored.ids.get("sharedid.org").map(|id| id.uid.as_str()), - Some(cookie_id), - "{source} should persist the returning-user EID cookie" + permit.then_some(cookie_id), + "{source} should persist the returning-user EID cookie only with a permit" ); } } @@ -1296,20 +1276,17 @@ mod tests { #[test] fn finalize_generated_ec_persists_eid_updates() { let settings = create_test_settings(); - let ec_id = sample_ec_id("geneid"); let graph = KvIdentityGraph::in_memory("test_store"); - let live = KvEntry::new( - &granting_consent(), + let mut ec_context = EcContext::new_for_test_with_ip( None, - current_timestamp(), - &settings.publisher.domain, + granting_consent(), + Some("192.0.2.10".to_owned()), ); - graph - .create(&ec_id, &live) - .expect("should seed generated row"); - let mut ec_context = - make_context(Some(&ec_id), None, false, true, Jurisdiction::NonRegulated); - ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + ec_context + .generate_if_needed(&settings, Some(&graph)) + .expect("should mint a new EC"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let ec_id = ec_context.ec_value().expect("should mint").to_owned(); let partners = vec![make_partner("sharedid.org")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); let mut response = empty_response(); @@ -1327,114 +1304,18 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should remain"); + .expect("should create the row"); assert_eq!( stored.ids.get("sharedid.org").map(|id| id.uid.as_str()), - Some("generated-cookie-id") - ); - } - - #[test] - fn eid_sync_measurement_dimensions_are_bounded_and_identity_free() { - let sources = [ - EidSyncSource::Navigation, - EidSyncSource::Auction, - EidSyncSource::PageBids, - EidSyncSource::NewEc, - ]; - let outcomes = [ - EidCookieSyncOutcome::AlreadyMatched, - EidCookieSyncOutcome::Written, - EidCookieSyncOutcome::WrittenWithDeferredFreshness, - EidCookieSyncOutcome::ConflictMatched, - EidCookieSyncOutcome::DeferredConflict, - EidCookieSyncOutcome::DeferredFreshness, - EidCookieSyncOutcome::DeferredStaleRead, - EidCookieSyncOutcome::Missing, - EidCookieSyncOutcome::ConsentWithdrawn, - EidCookieSyncOutcome::Failed, - ]; - - assert_eq!( - sources.map(|source| source.to_string()), - ["navigation", "auction", "page_bids", "new_ec"] - ); - assert_eq!( - outcomes.map(|outcome| outcome.to_string()), - [ - "already_matched", - "written", - "written_with_deferred_freshness", - "conflict_matched", - "deferred_conflict", - "deferred_freshness", - "deferred_stale_read", - "missing", - "consent_withdrawn", - "failed", - ] - ); - - assert_eq!( - EidSyncMeasurement::new( - EidSyncSource::Navigation, - EidCookieSyncOutcome::AlreadyMatched, - ), - EidSyncMeasurement { - source: EidSyncSource::Navigation, - outcome: EidCookieSyncOutcome::AlreadyMatched, - already_matched: 1, - written: 0, - conflict_duplicate: 0, - deferred: 0, - } - ); - assert_eq!( - EidSyncMeasurement::new( - EidSyncSource::Auction, - EidCookieSyncOutcome::WrittenWithDeferredFreshness, - ), - EidSyncMeasurement { - source: EidSyncSource::Auction, - outcome: EidCookieSyncOutcome::WrittenWithDeferredFreshness, - already_matched: 0, - written: 1, - conflict_duplicate: 0, - deferred: 1, - } - ); - assert_eq!( - EidSyncMeasurement::new(EidSyncSource::NewEc, EidCookieSyncOutcome::ConflictMatched,), - EidSyncMeasurement { - source: EidSyncSource::NewEc, - outcome: EidCookieSyncOutcome::ConflictMatched, - already_matched: 0, - written: 0, - conflict_duplicate: 1, - deferred: 0, - } - ); - assert_eq!( - EidSyncMeasurement::new( - EidSyncSource::PageBids, - EidCookieSyncOutcome::DeferredStaleRead, - ), - EidSyncMeasurement { - source: EidSyncSource::PageBids, - outcome: EidCookieSyncOutcome::DeferredStaleRead, - already_matched: 0, - written: 0, - conflict_duplicate: 0, - deferred: 1, - } + Some("generated-cookie-id"), + "should ingest the browser EID in the create" ); } #[test] - fn finalize_auction_transient_miss_still_persists_eid_updates() { - // `/auction` saves its first lookup into the context and is never - // recovery eligible, so a stale miss there has no later chance to - // retry. Finalization must revalidate before dropping collected IDs. + fn finalize_auction_transient_miss_does_not_persist_eid_updates() { + // `/auction` never writes KV (spec D1): browser EIDs are persisted only + // by the page load's navigation write, whatever its snapshot says. let settings = create_test_settings(); let ec_id = sample_ec_id("named1"); let graph = KvIdentityGraph::in_memory("test_store"); @@ -1473,21 +1354,18 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should remain"); - assert_eq!( - stored.ids.get("sharedid.org").map(|id| id.uid.as_str()), - Some("shared-cookie-id"), - "a stale endpoint miss must not suppress EID persistence" + .expect("should keep the row"); + assert!( + stored.ids.is_empty(), + "an auction request must leave the row unchanged" ); } #[test] - fn finalize_persists_every_configured_partner_from_client_eids_over_a_trimmed_cookie() { - // Regression test for #1184: the `ts-eids` cookie only carries what - // the browser could fit under its size cap, but `/auction` also - // hands finalization the full EID set from the request body via - // `EcContext::set_client_eids`. That full set must land in KV even - // when the cookie alone would have dropped a configured partner. + fn finalize_never_persists_auction_body_eids() { + // `/auction` body EIDs go to bidders on that request only and are + // never persisted (spec §5.1); neither is the `ts-eids` cookie on a + // non-navigation request. let settings = create_test_settings(); let ec_id = sample_ec_id("cleids1"); let graph = KvIdentityGraph::in_memory("test_store"); @@ -1517,22 +1395,22 @@ mod tests { .expect("should serialize test cookie payload"), ); ec_context.set_client_eids(vec![ - Eid { - source: "id5-sync.com".to_owned(), - uids: vec![Uid { + Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { id: "ID5_from_body".to_owned(), atype: Some(1), ext: None, }], - }, - Eid { - source: "liveramp.com".to_owned(), - uids: vec![Uid { + ), + Eid::new( + "liveramp.com".to_owned(), + vec![Uid { id: "LR_from_body".to_owned(), atype: Some(3), ext: None, }], - }, + ), ]); let partners = vec![make_partner("id5-sync.com"), make_partner("liveramp.com")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); @@ -1551,16 +1429,10 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should exist after ingestion"); - assert_eq!( - stored.ids.get("id5-sync.com").map(|id| id.uid.as_str()), - Some("ID5_from_body"), - "the request body's EID should win over the cookie's stale value" - ); - assert_eq!( - stored.ids.get("liveramp.com").map(|id| id.uid.as_str()), - Some("LR_from_body"), - "a partner the cookie trimmed must still be ingested from the body" + .expect("should keep the row"); + assert!( + stored.ids.is_empty(), + "should persist neither the auction body nor the cookie EIDs" ); } @@ -1591,18 +1463,17 @@ mod tests { false, purpose1_only_consent, ); - ec_context.set_kv_snapshot(EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }); - ec_context.set_eid_sync_source(EidSyncSource::Auction); - ec_context.set_client_eids(vec![Eid { - source: "id5-sync.com".to_owned(), - uids: vec![Uid { + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + ec_context.set_eid_sync_source(EidSyncSource::Navigation); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + ec_context.set_client_eids(vec![Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { id: "ID5_should_not_persist".to_owned(), atype: Some(1), ext: None, }], - }]); + )]); let partners = vec![make_partner("id5-sync.com")]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); let mut response = empty_response(); @@ -1620,7 +1491,7 @@ mod tests { let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("row should remain"); + .expect("should keep the row"); assert!( !stored.ids.contains_key("id5-sync.com"), "denying TCF Purpose 4 must keep the EID out of KV even though Purpose 1 \ @@ -1629,38 +1500,147 @@ mod tests { } #[test] - fn finalize_auction_confirmed_miss_does_not_create_a_row() { - // The same path with a genuinely absent row must stay a no-op: a route - // without orphan recovery must never mint an identity-graph entry. + fn finalize_withholds_provider_owned_fallback_when_purpose_four_is_denied() { + // Arrange let settings = create_test_settings(); - let ec_id = sample_ec_id("named2"); + let ec_id = sample_ec_id("purp4o"); let graph = KvIdentityGraph::in_memory("test_store"); - let mut ec_context = returning_user_context( - &ec_id, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - false, - ); - ec_context.set_eid_sync_source(EidSyncSource::Auction); - let partners = vec![make_partner("sharedid.org")]; - let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); - let mut response = empty_response(); - - ec_finalize_response( - &settings, - &mut ec_context, - Some(&graph), - ®istry, + let purpose1_only_consent = purpose_one_only_consent(); + let live = KvEntry::new( + &purpose1_only_consent, None, - Some("shared-cookie-id"), - &mut response, - ); - - assert_did_not_rotate(&ec_context, &ec_id, &response); - assert!( - graph.get(&ec_id).expect("should read store").is_none(), - "a confirmed miss must not create a root entry" + current_timestamp(), + &settings.publisher.domain, + ); + graph + .create(&ec_id, &live) + .expect("should seed the live row this request updates"); + let mut ec_context = EcContext::new_for_test_with_cookie( + Some(ec_id.clone()), + Some(ec_id.clone()), + true, + false, + purpose1_only_consent, + ); + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + ec_context.set_eid_sync_source(EidSyncSource::Navigation); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + ec_context.set_client_eids(vec![Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { + id: "ID5_should_not_persist".to_owned(), + atype: Some(1), + ext: None, + }], + )]); + let mut registry = PartnerRegistry::from_config(&[make_partner("id5-sync.com")]) + .expect("should build registry"); + assert!( + registry.mark_provider_owned("id5-sync.com", None), + "should mark the provider source" + ); + let mut response = empty_response(); + + // Act + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry, + None, + None, + &mut response, + ); + + // Assert + let (stored, _) = graph + .get(&ec_id) + .expect("should read store") + .expect("should keep the row"); + assert!( + !stored.ids.contains_key("id5-sync.com"), + "should keep the provider-owned fallback out of KV without Purpose 4" + ); + } + + #[test] + fn finalize_skips_browser_uid_for_provider_owned_source() { + // Arrange + let settings = create_test_settings(); + let graph = KvIdentityGraph::in_memory("test_store"); + let mut ec_context = EcContext::new_for_test_with_ip( + None, + granting_consent(), + Some("192.0.2.10".to_owned()), + ); + ec_context + .generate_if_needed(&settings, Some(&graph)) + .expect("should mint a new EC"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let ec_id = ec_context.ec_value().expect("should mint").to_owned(); + let mut registry = PartnerRegistry::from_config(&[make_partner("sharedid.org")]) + .expect("should build registry"); + assert!( + registry.mark_provider_owned("sharedid.org", None), + "should mark the provider source" + ); + let mut response = empty_response(); + + // Act + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry, + None, + Some("fallback-cookie-id"), + &mut response, + ); + + // Assert + let (stored, _) = graph + .get(&ec_id) + .expect("should read store") + .expect("should create the row"); + assert!( + !stored.ids.contains_key("sharedid.org"), + "should not store a browser UID for a provider-owned source" + ); + } + + #[test] + fn finalize_auction_confirmed_miss_does_not_create_a_row() { + // The same path with a genuinely absent row must stay a no-op: a route + // without orphan recovery must never mint an identity-graph entry. + let settings = create_test_settings(); + let ec_id = sample_ec_id("named2"); + let graph = KvIdentityGraph::in_memory("test_store"); + let mut ec_context = returning_user_context( + &ec_id, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, + false, + ); + ec_context.set_eid_sync_source(EidSyncSource::Auction); + let partners = vec![make_partner("sharedid.org")]; + let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry, + None, + Some("shared-cookie-id"), + &mut response, + ); + + assert_did_not_rotate(&ec_context, &ec_id, &response); + assert!( + graph.get(&ec_id).expect("should read store").is_none(), + "a confirmed miss must not create a root entry" ); } @@ -1870,6 +1850,7 @@ mod tests { ec_id: generated_ec, }); let graph = KvIdentityGraph::in_memory("test_store"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); ec_finalize_response( @@ -1978,6 +1959,11 @@ mod tests { Some("192.0.2.10".to_owned()), ); ec.set_recovery_eligible(recovery_eligible); + if recovery_eligible { + // Only a top-level navigation is recovery eligible, and only a + // navigation carries the page-load write permit. + ec.set_navigation_permit(NavigationPermit::for_test()); + } ec.set_kv_snapshot(snapshot); ec } @@ -2121,6 +2107,7 @@ mod tests { .create(&active_ec, &live_entry()) .expect("should seed active row"); ec_context.set_kv_snapshot(graph.load_snapshot(&active_ec)); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); ec_finalize_response( @@ -2148,7 +2135,7 @@ mod tests { } #[test] - fn finalize_withdrawal_tombstones_both_present_ids_once() { + fn finalize_withdrawal_tombstones_the_active_and_cookie_ids_once() { let settings = create_test_settings(); let active_ec = sample_ec_id("activ3"); let cookie_ec = sample_ec_id("cook3e"); @@ -2174,6 +2161,7 @@ mod tests { ) .expect("should seed cookie row"); ec_context.set_kv_snapshot(graph.load_snapshot(&active_ec)); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut response = empty_response(); ec_finalize_response( @@ -2204,13 +2192,10 @@ mod tests { ); assert!( !cookie_tombstone.consent.ok, - "cookie row should be withdrawn" - ); - assert!( - cookie_tombstone.ids.is_empty(), - "cookie IDs should be cleared" + "the cookie's row should be withdrawn too" ); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let mut repeated_response = empty_response(); ec_finalize_response( &settings, @@ -2238,12 +2223,12 @@ mod tests { .expect("should retain cookie tombstone") .1, cookie_generation, - "repeated finalization should not rewrite cookie tombstone" + "repeated finalization should not rewrite the cookie tombstone" ); } #[test] - fn finalize_withdrawal_keeps_cookie_deletion_on_kv_failure() { + fn finalize_withdrawal_keeps_ec_cookie_but_expires_marker_on_kv_failure() { let settings = create_test_settings(); let ec_id = sample_ec_id("failw1"); let consent = ConsentContext { @@ -2257,6 +2242,7 @@ mod tests { ec_context.set_pull_sync_marker_for_test( crate::ec::pull_sync_marker::PullSyncMarkerState::Invalid, ); + ec_context.set_navigation_permit(NavigationPermit::for_test()); let graph = KvIdentityGraph::failing("unavailable-store"); let mut response = empty_response(); @@ -2282,10 +2268,8 @@ mod tests { "KV failure should not change response status" ); assert!( - cookies - .iter() - .any(|cookie| { cookie.starts_with("ts-ec=;") && cookie.contains("Max-Age=0") }), - "KV failure should not prevent EC cookie deletion" + cookies.iter().all(|cookie| !cookie.starts_with("ts-ec=;")), + "KV failure should keep the EC cookie so a later navigation can tombstone the row" ); assert!( cookies.iter().any(|cookie| { @@ -2315,9 +2299,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "partner-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("partner-uid".to_owned()), ); ec_context.set_kv_snapshot(EcKvSnapshot::Present { ec_id, @@ -2421,9 +2403,531 @@ mod tests { ); } + fn registry_for(source: &str) -> PartnerRegistry { + PartnerRegistry::from_config(&[make_partner(source)]).expect("should build registry") + } + + fn eids_cookie(source: &str, uid: &str) -> String { + base64::engine::general_purpose::STANDARD.encode( + serde_json::json!([{ "source": source, "uids": [{ "id": uid, "atype": 1 }] }]) + .to_string(), + ) + } + + #[test] + fn auction_request_never_writes_browser_eids() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("auc001"); + let graph = KvIdentityGraph::in_memory("finalize"); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let mut ec_context = EcContext::new_for_test(Some(ec_id.clone()), granting_consent()); + ec_context.set_eid_sync_source(EidSyncSource::Auction); + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + let cookie = eids_cookie("partner.example.com", "u-1"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry_for("partner.example.com"), + Some(&cookie), + None, + &mut response, + ); + + let (entry, _) = graph + .get(&ec_id) + .expect("should read") + .expect("should exist"); + assert!( + entry.ids.is_empty(), + "should not write from a non-navigation request" + ); + } + + #[test] + fn navigation_writes_browser_eids_once() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("nav001"); + let graph = KvIdentityGraph::in_memory("finalize"); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let mut ec_context = EcContext::new_for_test(Some(ec_id.clone()), granting_consent()); + ec_context.set_eid_sync_source(EidSyncSource::Navigation); + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let cookie = eids_cookie("partner.example.com", "u-1"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry_for("partner.example.com"), + Some(&cookie), + None, + &mut response, + ); + + let (entry, _) = graph + .get(&ec_id) + .expect("should read") + .expect("should exist"); + assert_eq!( + entry.ids["partner.example.com"].uid, "u-1", + "should persist browser uid" + ); + assert!( + ec_context.take_navigation_permit().is_none(), + "should spend the permit" + ); + } + + #[test] + fn redirect_navigation_does_not_write() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("red001"); + let graph = KvIdentityGraph::in_memory("finalize"); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let mut ec_context = EcContext::new_for_test(Some(ec_id.clone()), granting_consent()); + ec_context.set_eid_sync_source(EidSyncSource::Navigation); + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let cookie = eids_cookie("partner.example.com", "u-1"); + let mut response = empty_response(); + *response.status_mut() = http::StatusCode::FOUND; + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry_for("partner.example.com"), + Some(&cookie), + None, + &mut response, + ); + + let (entry, _) = graph + .get(&ec_id) + .expect("should read") + .expect("should exist"); + assert!(entry.ids.is_empty(), "should not write on a redirect"); + } + + #[test] + fn generated_ec_without_permit_gets_no_cookie_or_row() { + let settings = create_test_settings(); + let graph = KvIdentityGraph::in_memory("finalize"); + let mut ec_context = EcContext::new_for_test_with_ip( + None, + granting_consent(), + Some("192.0.2.10".to_owned()), + ); + ec_context + .generate_if_needed(&settings, Some(&graph)) + .expect("should mint"); + let ec_id = ec_context.ec_value().expect("should mint").to_owned(); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert!( + graph.get(&ec_id).expect("should read").is_none(), + "should not create without a permit" + ); + assert!( + get_header(&response, "set-cookie").is_none(), + "should not issue the EC" + ); + } + + #[test] + fn generated_ec_with_permit_creates_row_and_sets_cookie() { + let settings = create_test_settings(); + let graph = KvIdentityGraph::in_memory("finalize"); + let mut ec_context = EcContext::new_for_test_with_ip( + None, + granting_consent(), + Some("192.0.2.10".to_owned()), + ); + ec_context + .generate_if_needed(&settings, Some(&graph)) + .expect("should mint"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let ec_id = ec_context.ec_value().expect("should mint").to_owned(); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert!( + graph.get(&ec_id).expect("should read").is_some(), + "should create the row" + ); + assert!( + get_header(&response, "set-cookie").is_some(), + "should issue the EC" + ); + } + + #[test] + fn generated_ec_collision_is_not_retried_or_issued() { + let settings = create_test_settings(); + let graph = KvIdentityGraph::in_memory("finalize"); + let mut ec_context = EcContext::new_for_test_with_ip( + None, + granting_consent(), + Some("192.0.2.10".to_owned()), + ); + ec_context + .generate_if_needed(&settings, Some(&graph)) + .expect("should mint"); + let ec_id = ec_context.ec_value().expect("should mint").to_owned(); + graph + .create(&ec_id, &live_entry()) + .expect("should seed a colliding row"); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert!( + get_header(&response, "set-cookie").is_none(), + "should not issue an EC whose row was not created" + ); + } + + fn withdrawing_context(ec_id: &str) -> EcContext { + let consent = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + source: ConsentSource::Cookie, + ..Default::default() + }; + make_context_with_consent(Some(ec_id), Some(ec_id), true, false, consent) + } + + fn assert_withdrawn(graph: &KvIdentityGraph, ec_id: &str, response: &Response) { + let (entry, _) = graph + .get(ec_id) + .expect("should read") + .expect("should keep the tombstone"); + assert!(!entry.consent.ok, "should tombstone the row"); + assert!( + get_header_str(response, "set-cookie") + .is_some_and(|cookie| cookie.contains("Max-Age=0")), + "should expire the EC cookie" + ); + } + + #[test] + fn withdrawal_without_permit_tombstones_row_and_expires_cookie() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("wd0001"); + let graph = KvIdentityGraph::in_memory("finalize"); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let mut ec_context = withdrawing_context(&ec_id); + let mut response = empty_response(); + set_header(&mut response, "x-ts-ec", "stale"); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert_withdrawn(&graph, &ec_id, &response); + assert!( + get_header(&response, "x-ts-ec").is_none(), + "should still strip EC response headers" + ); + } + + #[test] + fn withdrawal_on_a_redirect_tombstones_row_and_expires_cookie() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("wd0002"); + let graph = KvIdentityGraph::in_memory("finalize"); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let mut ec_context = withdrawing_context(&ec_id); + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + ec_context.set_navigation_permit(NavigationPermit::for_test()); + let mut response = empty_response(); + *response.status_mut() = http::StatusCode::FOUND; + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert_withdrawn(&graph, &ec_id, &response); + } + + #[test] + fn withdrawal_retries_after_a_concurrent_write() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("wd0003"); + let graph = KvIdentityGraph::in_memory("finalize"); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let mut ec_context = withdrawing_context(&ec_id); + ec_context.set_kv_snapshot(graph.load_snapshot(&ec_id)); + graph + .upsert_partner_id(&ec_id, "partner.example.com", "late-uid") + .expect("should write concurrently after the snapshot"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert_withdrawn(&graph, &ec_id, &response); + } + fn live_entry() -> KvEntry { let mut entry = KvEntry::tombstone(1000); entry.consent.ok = true; entry } + + #[test] + fn finalize_withholds_after_response_lookups_when_the_confirmed_write_fails() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("after3"); + let graph = KvIdentityGraph::write_failing_with( + &ec_id, + &crate::ec::kv_types::KvEntry::minimal("p.example", "uid", 1), + ); + // The preload missed a row the confirmation read then finds, and the + // navigation's write against it fails. + let mut ec_context = returning_user_context( + &ec_id, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, + true, + ); + ec_context.set_eid_sync_source(EidSyncSource::Navigation); + let cookie = eids_cookie("partner.example.com", "u-1"); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry_for("partner.example.com"), + Some(&cookie), + None, + &mut response, + ); + + assert!( + ec_context.take_after_response_permit().is_none(), + "should not allow a follow-up write after the page-load write failed" + ); + } + + #[test] + fn finalize_allows_after_response_lookups_on_a_live_row() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("after1"); + let graph = KvIdentityGraph::in_memory("test_store"); + graph + .create( + &ec_id, + &crate::ec::kv_types::KvEntry::minimal("p.example", "uid", 1), + ) + .expect("should seed the identity"); + let mut ec_context = returning_user_context(&ec_id, graph.load_snapshot(&ec_id), true); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert!( + ec_context.take_after_response_permit().is_some(), + "should allow after-response lookups when the page write had nothing to change" + ); + } + + #[test] + fn finalize_withholds_after_response_lookups_when_the_row_read_failed() { + let settings = create_test_settings(); + let ec_id = sample_ec_id("after2"); + let graph = KvIdentityGraph::in_memory("test_store"); + let mut ec_context = returning_user_context( + &ec_id, + EcKvSnapshot::Failed { + ec_id: ec_id.clone(), + }, + true, + ); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert!( + ec_context.take_after_response_permit().is_none(), + "should not allow after-response lookups when the page phase failed" + ); + } + + #[test] + fn finalize_withholds_after_response_lookups_on_a_tombstoned_row() { + // Arrange: a consented navigation reads a withdrawn row. + let settings = create_test_settings(); + let ec_id = sample_ec_id("after4"); + let graph = KvIdentityGraph::in_memory("test_store"); + graph + .create(&ec_id, &KvEntry::tombstone(1000)) + .expect("should seed the tombstone"); + let mut ec_context = returning_user_context(&ec_id, graph.load_snapshot(&ec_id), true); + let mut response = empty_response(); + + // Act + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + // Assert + assert!( + ec_context.take_after_response_permit().is_none(), + "should not allow after-response lookups for a withdrawn row" + ); + } + + #[test] + fn finalize_adopted_row_keeps_a_concurrent_provider_opt_out() { + // Arrange: the preload missed the row; meanwhile another request + // recorded provider P's opt-out on it, and P's lookup found a UID. + let settings = create_test_settings(); + let ec_id = sample_ec_id("optout"); + let graph = KvIdentityGraph::in_memory("test_store"); + let provider = ProviderId::new("example-provider"); + let mut live = live_entry(); + live.opt_outs + .insert(provider.clone(), current_timestamp() + 86_400); + graph.create(&ec_id, &live).expect("should seed the row"); + let mut ec_context = returning_user_context( + &ec_id, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, + true, + ); + let uid = KvUid::new( + UidValue::try_from("p-restored").expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(provider.clone()), + Expiry::Never, + ); + ec_context.set_lookup_results(LookupResults::new( + &ec_id, + vec![ProviderLookup { + provider, + status: LookupStatus::Found { + kept: 1, + dropped: DropCounts::default(), + }, + entries: vec![( + SourceDomain::try_from("provider.example.com").expect("should build source"), + vec![KvEid::new(EidFields::default(), vec![uid])], + )], + cleared: Vec::new(), + }], + )); + let mut response = empty_response(); + + // Act + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry_for("provider.example.com"), + None, + None, + &mut response, + ); + + // Assert + let (entry, _) = graph + .get(&ec_id) + .expect("should read row") + .expect("should keep row"); + assert!( + !entry.ids.contains_key("provider.example.com"), + "should not restore the opted-out provider's UID" + ); + } } diff --git a/crates/trusted-server-core/src/ec/identify.rs b/crates/trusted-server-core/src/ec/identify.rs index 94464d1a57..ed36371bbd 100644 --- a/crates/trusted-server-core/src/ec/identify.rs +++ b/crates/trusted-server-core/src/ec/identify.rs @@ -12,13 +12,15 @@ use url::Url; use super::auth::authenticate_bearer; use super::consent::ec_consent_granted; use crate::error::TrustedServerError; -use crate::openrtb::{Eid, Uid}; +use crate::openrtb::Eid; use crate::settings::Settings; -use super::EcContext; +use super::eid_types::UnixSeconds; +use super::eids::source_views; use super::kv::KvIdentityGraph; use super::log_id; use super::registry::PartnerRegistry; +use super::{EcContext, current_timestamp}; /// Handles `GET /_ts/api/v1/identify`. /// @@ -84,7 +86,7 @@ pub fn handle_identify( }; let mut degraded = false; - let mut uid: Option = None; + let mut eid: Option = None; let mut cluster_size: Option = None; match kv.get(ec_id) { @@ -95,12 +97,20 @@ pub fn handle_identify( // write back with the live-entry TTL. log::trace!("Identify found tombstone for '{}'", log_id(ec_id)); } else { - // Extract only this partner's UID. - if let Some(partner_uid) = entry.ids.get(&partner.source_domain) - && !partner_uid.uid.is_empty() - { - uid = Some(partner_uid.uid.clone()); - } + // Extract only this partner's first live entry. A + // provider-owned source is withheld while its owner lacks TCF + // vendor consent, as in the bidstream. + let now = UnixSeconds::new(current_timestamp()); + let vendor_denied = partner.owner_vendor_consent_denied(ec_context.consent()); + eid = entry + .ids + .get(&partner.source_domain) + .filter(|_| !vendor_denied) + .and_then(|record| { + source_views(&partner.source_domain, record, partner.openrtb_atype, now) + .next() + }) + .map(Eid::from); // Evaluate cluster size lazily for identify responses. Existing // stored cluster_size values are reused without a prefix-list call. @@ -124,14 +134,10 @@ pub fn handle_identify( } } - let eid = uid.as_ref().map(|u| Eid { - source: partner.source_domain.clone(), - uids: vec![Uid { - id: u.clone(), - atype: Some(partner.openrtb_atype), - ext: None, - }], - }); + let uid = eid + .as_ref() + .and_then(|eid| eid.uids.first()) + .map(|uid| uid.id.clone()); let body = IdentifyResponse { ec: ec_id.to_owned(), @@ -334,10 +340,13 @@ mod tests { use super::*; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ConsentContext, ConsentSource}; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, UidValue, UnixSeconds}; + use crate::ec::kv_types::{KvEid, KvEntry, KvPartnerId, KvUid}; use crate::ec::registry::PartnerRegistry; use crate::redacted::Redacted; use crate::settings::EcPartner; - use crate::test_support::tests::create_test_settings; + use crate::test_support::tests::{consent_granting_vendors, create_test_settings}; + use std::num::NonZeroU16; const VALID_API_TOKEN: &str = "identify-test-token-32-bytes-min"; @@ -756,4 +765,115 @@ mod tests { "should vary on identity request inputs for preflight" ); } + + fn provider_record(id: &str, expiry: Expiry) -> KvPartnerId { + let uid = KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + expiry, + ); + let fields = EidFields { + inserter: Some("inserter.example.com".into()), + ..EidFields::default() + }; + KvPartnerId::from_eids(vec![KvEid::new(fields, vec![uid])], UnixSeconds::new(0)) + .expect("should build record") + } + + fn identify_body_for_record(record: KvPartnerId) -> serde_json::Value { + let registry = + PartnerRegistry::from_config(&[make_test_partner("ssp.example.com", VALID_API_TOKEN)]) + .expect("should build registry"); + let consent = ConsentContext { + jurisdiction: Jurisdiction::NonRegulated, + source: ConsentSource::Cookie, + ..ConsentContext::default() + }; + identify_body_with(record, ®istry, consent) + } + + fn identify_body_with( + record: KvPartnerId, + registry: &PartnerRegistry, + consent: ConsentContext, + ) -> serde_json::Value { + let settings = create_test_settings(); + let kv = KvIdentityGraph::in_memory("identify_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let mut entry = KvEntry::minimal("other.example.com", "other-uid", 1); + entry.ids.insert("ssp.example.com".to_owned(), record); + kv.create(&ec_id, &entry).expect("should seed KV entry"); + let req = Request::builder() + .method("GET") + .uri("https://edge.example.com/identify") + .header("authorization", format!("Bearer {VALID_API_TOKEN}")) + .body(EdgeBody::empty()) + .expect("should build test request"); + let ec_context = EcContext::new_for_test(Some(ec_id), consent); + + let response = handle_identify(&settings, &kv, registry, &req, &ec_context) + .expect("should construct identify response"); + + assert_eq!(response.status(), StatusCode::OK, "should return 200"); + serde_json::from_slice::( + &response.into_body().into_bytes().unwrap_or_default(), + ) + .expect("should decode identify response JSON") + } + + #[test] + fn handle_identify_withholds_a_source_whose_owner_lacks_vendor_consent() { + let mut registry = + PartnerRegistry::from_config(&[make_test_partner("ssp.example.com", VALID_API_TOKEN)]) + .expect("should build registry"); + assert!( + registry.mark_provider_owned("ssp.example.com", NonZeroU16::new(9999)), + "should mark the source provider-owned" + ); + let record = || provider_record("live", Expiry::Never); + + let denied = identify_body_with(record(), ®istry, consent_granting_vendors(vec![1])); + let granted = identify_body_with(record(), ®istry, consent_granting_vendors(vec![9999])); + + assert!( + denied.get("uid").is_none(), + "should withhold the UID without the owner's vendor consent" + ); + assert_eq!( + granted["uid"], "live", + "should return the UID with the owner's vendor consent" + ); + } + + #[test] + fn handle_identify_omits_expired_provider_uid() { + let body = + identify_body_for_record(provider_record("old", Expiry::At(UnixSeconds::new(1)))); + + assert!( + body.get("uid").is_none(), + "should omit expired provider uid" + ); + assert!( + body.get("eid").is_none(), + "should omit eid without a live uid" + ); + } + + #[test] + fn handle_identify_returns_live_provider_eid_with_eid_fields() { + let body = identify_body_for_record(provider_record("live", Expiry::Never)); + + assert_eq!(body["uid"], "live", "should return live provider uid"); + assert_eq!( + body["eid"]["inserter"], "inserter.example.com", + "should keep provider EID-level fields" + ); + assert_eq!( + body["eid"]["uids"][0]["atype"], 1, + "should use the provider atype" + ); + } } diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index b0eb83d3fa..0a35fd8baa 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -21,7 +21,8 @@ use crate::error::TrustedServerError; use super::generation::ec_hash; use super::kv_backend::{EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteMode, EcKvWriteOutcome}; -use super::kv_types::{KvEntry, KvMetadata, KvNetwork}; +use super::kv_types::{KvEntry, KvMetadata, KvNetwork, MAX_ENTRY_BYTES}; +use super::page_load::{AfterResponseWrite, Generation, KvWriteError, PageLoadWrite, RowChange}; use super::{EcKvSnapshot, checked_current_timestamp, current_timestamp, log_id}; /// Maximum number of CAS retry attempts before giving up. @@ -59,29 +60,31 @@ pub enum UpsertResult { ConsentWithdrawn, /// The partner ID already had the requested UID, so no write was needed. Unchanged, + /// The row with this partner ID would exceed [`MAX_ENTRY_BYTES`], so + /// nothing was written. Retrying gives the same result. + RowFull, } -/// Outcome of atomically creating an identity-graph root when absent. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum CreateIfAbsentOutcome { - /// The candidate entry was persisted. - Written, - /// A row already exists for the candidate key. - AlreadyExists, +/// A serialized entry, or the size that kept it from being written. +enum SerializedEntry { + /// Body and metadata ready to write. + Fits { body: String, meta: String }, + /// The body is this many bytes, over [`MAX_ENTRY_BYTES`]. + TooLarge(usize), } /// Partner UID update to apply to a KV identity graph entry. #[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct PartnerIdUpdate { - /// Partner namespace key in [`KvEntry::ids`]. - pub(crate) partner_id: String, - /// Partner-scoped user ID value. - pub(crate) uid: String, +pub struct PartnerIdUpdate { + /// Source domain key in [`KvEntry::ids`]. + pub partner_id: String, + /// UID to fill in. + pub uid: String, } impl PartnerIdUpdate { /// Creates a partner UID update. - pub(crate) fn new(partner_id: impl Into, uid: impl Into) -> Self { + pub fn new(partner_id: impl Into, uid: impl Into) -> Self { Self { partner_id: partner_id.into(), uid: uid.into(), @@ -89,104 +92,6 @@ impl PartnerIdUpdate { } } -/// Terminal result of one browser EID-cookie persistence attempt. -#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] -pub(crate) enum EidCookieSyncOutcome { - /// The stored values already matched without a write. - #[display("already_matched")] - AlreadyMatched, - /// The one conditional write succeeded. - #[display("written")] - Written, - /// The write added missing IDs while deferring different values of unknown freshness. - #[display("written_with_deferred_freshness")] - WrittenWithDeferredFreshness, - /// A conflicting writer persisted every desired value. - #[display("conflict_matched")] - ConflictMatched, - /// A conflict left at least one desired value absent or different. - #[display("deferred_conflict")] - DeferredConflict, - /// A different stored value had unknown freshness. - #[display("deferred_freshness")] - DeferredFreshness, - /// An eventually consistent read missed a row already proven to exist. - #[display("deferred_stale_read")] - DeferredStaleRead, - /// The identity graph row was missing. - #[display("missing")] - Missing, - /// Consent had been withdrawn in the identity graph. - #[display("consent_withdrawn")] - ConsentWithdrawn, - /// KV or serialization failed. - #[display("failed")] - Failed, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum CookieUpdateApplication { - AlreadyMatched, - Changed { deferred: bool }, - DeferredFreshness, -} - -fn apply_cookie_partner_id_updates( - entry: &mut KvEntry, - updates: &[PartnerIdUpdate], -) -> CookieUpdateApplication { - let mut latest_updates = BTreeMap::new(); - for update in updates { - latest_updates.insert(update.partner_id.as_str(), update.uid.as_str()); - } - - let mut changed = false; - let mut deferred = false; - for (partner_id, uid) in latest_updates { - match entry.ids.get(partner_id) { - Some(existing) if existing.uid == uid => continue, - // Browser EID cookies carry no value-owned sequence or timestamp. - // A different value therefore has unknown freshness and must not - // replace the stored value. - Some(_) => { - deferred = true; - continue; - } - None => {} - } - - entry.ids.insert( - partner_id.to_owned(), - super::kv_types::KvPartnerId { - uid: uid.to_owned(), - }, - ); - changed = true; - } - - if changed { - CookieUpdateApplication::Changed { deferred } - } else if deferred { - CookieUpdateApplication::DeferredFreshness - } else { - CookieUpdateApplication::AlreadyMatched - } -} - -fn partner_id_updates_match(entry: &KvEntry, updates: &[PartnerIdUpdate]) -> bool { - let mut latest_updates = BTreeMap::new(); - for update in updates { - latest_updates.insert(update.partner_id.as_str(), update.uid.as_str()); - } - - latest_updates.into_iter().all(|(partner_id, uid)| { - entry - .ids - .get(partner_id) - .is_some_and(|existing| existing.uid == uid) - }) -} - pub(crate) fn apply_partner_id_updates(entry: &mut KvEntry, updates: &[PartnerIdUpdate]) -> bool { let mut latest_updates = BTreeMap::new(); for update in updates { @@ -205,9 +110,7 @@ pub(crate) fn apply_partner_id_updates(entry: &mut KvEntry, updates: &[PartnerId entry.ids.insert( partner_id.to_owned(), - super::kv_types::KvPartnerId { - uid: uid.to_owned(), - }, + super::kv_types::KvPartnerId::single(uid.to_owned()), ); changed = true; } @@ -269,10 +172,30 @@ impl KvIdentityGraph { } /// Serializes an entry body and metadata for insertion. + /// + /// A body over [`MAX_ENTRY_BYTES`] is an error; callers that report a + /// full row separately use [`Self::serialize_entry_unless_full`]. fn serialize_entry( entry: &KvEntry, store_name: &str, ) -> Result<(String, String), Report> { + match Self::serialize_entry_unless_full(entry, store_name)? { + SerializedEntry::Fits { body, meta } => Ok((body, meta)), + SerializedEntry::TooLarge(bytes) => Err(Report::new(TrustedServerError::KvStore { + store_name: store_name.to_owned(), + message: format!( + "Refusing to write KV entry of {bytes} bytes (limit {MAX_ENTRY_BYTES})" + ), + })), + } + } + + /// Serializes an entry body and metadata, reporting an oversized body as + /// [`SerializedEntry::TooLarge`] rather than an error. + fn serialize_entry_unless_full( + entry: &KvEntry, + store_name: &str, + ) -> Result> { entry.validate().map_err(|message| { Report::new(TrustedServerError::KvStore { store_name: store_name.to_owned(), @@ -284,13 +207,19 @@ impl KvIdentityGraph { store_name: store_name.to_owned(), message: "Failed to serialize KV entry body".to_owned(), })?; + if body.len() > MAX_ENTRY_BYTES { + return Ok(SerializedEntry::TooLarge(body.len())); + } let meta = KvMetadata::from_entry(entry); let meta_str = serde_json::to_string(&meta).change_context(TrustedServerError::KvStore { store_name: store_name.to_owned(), message: "Failed to serialize KV entry metadata".to_owned(), })?; - Ok((body, meta_str)) + Ok(SerializedEntry::Fits { + body, + meta: meta_str, + }) } /// Reads the raw stored body, metadata, and generation for an EC ID key. @@ -357,12 +286,20 @@ impl KvIdentityGraph { ec_id: &str, body_bytes: &[u8], ) -> Result> { - let entry: KvEntry = + let mut entry: KvEntry = serde_json::from_slice(body_bytes).change_context(TrustedServerError::KvStore { store_name: store_name.to_owned(), message: format!("Failed to deserialize entry for key '{}'", log_id(ec_id)), })?; + let dropped = entry.drop_unreadable_ids(); + if dropped > 0 { + log::warn!( + "Dropped {dropped} unreadable partner IDs loading key '{}'", + log_id(ec_id) + ); + } + entry.validate().map_err(|message| { Report::new(TrustedServerError::KvStore { store_name: store_name.to_owned(), @@ -423,23 +360,6 @@ impl KvIdentityGraph { } } - /// Atomically creates an entry while preserving collision as normal control flow. - /// - /// # Errors - /// - /// Returns [`TrustedServerError::KvStore`] when serialization or store I/O fails. - pub fn create_if_absent( - &self, - ec_id: &str, - entry: &KvEntry, - ) -> Result> { - let (body, meta_str) = Self::serialize_entry(entry, self.store_name())?; - match self.write_entry(ec_id, &body, &meta_str, ENTRY_TTL, EcKvWriteMode::Add)? { - EcKvWriteOutcome::Written => Ok(CreateIfAbsentOutcome::Written), - EcKvWriteOutcome::PreconditionFailed => Ok(CreateIfAbsentOutcome::AlreadyExists), - } - } - /// Low-level write with shared error context. fn write_entry( &self, @@ -460,6 +380,85 @@ impl KvIdentityGraph { ) } + /// Applies the navigation's single row change in one attempt. + /// + /// The `permit` is consumed, so a navigation cannot write twice. Consent + /// withdrawals do not go through here: their tombstone is a privacy write + /// on any request (`tombstone_existing_from_snapshot`). + /// + /// # Errors + /// + /// Returns [`KvWriteError::Conflict`] when the row changed since it was + /// read, [`KvWriteError::AlreadyExists`] when a create finds a row, or + /// [`KvWriteError::Store`] on a serialization or store failure, including + /// a row over [`MAX_ENTRY_BYTES`]. Callers must not retry. + pub fn commit_page_load( + &self, + permit: PageLoadWrite, + ec_id: &str, + change: RowChange, + ) -> Result> { + permit.spend(); + let (entry, mode) = match change { + RowChange::Create(entry) => (entry, EcKvWriteMode::Add), + RowChange::Update { entry, expected } => { + (entry, EcKvWriteMode::IfGenerationMatch(expected.get())) + } + }; + self.write_row(ec_id, &entry, ENTRY_TTL, mode)?; + Ok(entry) + } + + /// Applies a navigation's follow-up update after its response was sent. + /// + /// The row is replaced only while it is still at `expected`, so a + /// concurrent write is never overwritten. The `permit` is consumed. + /// + /// # Errors + /// + /// Returns [`KvWriteError::Conflict`] when the row changed since it was + /// read, or [`KvWriteError::Store`] on a serialization or store failure, + /// including a row over [`MAX_ENTRY_BYTES`]. Callers must not retry. + pub fn commit_after_response( + &self, + permit: AfterResponseWrite, + ec_id: &str, + entry: &KvEntry, + expected: Generation, + ) -> Result<(), Report> { + permit.spend(); + self.write_row( + ec_id, + entry, + ENTRY_TTL, + EcKvWriteMode::IfGenerationMatch(expected.get()), + ) + } + + /// Serializes `entry` and writes it with `mode`, mapping a failed + /// precondition to [`KvWriteError::AlreadyExists`] for a create and + /// [`KvWriteError::Conflict`] otherwise. + fn write_row( + &self, + ec_id: &str, + entry: &KvEntry, + ttl: Duration, + mode: EcKvWriteMode, + ) -> Result<(), Report> { + let (body, metadata) = + Self::serialize_entry(entry, self.store_name()).change_context(KvWriteError::Store)?; + let outcome = self + .write_entry(ec_id, &body, &metadata, ttl, mode) + .change_context(KvWriteError::Store)?; + match outcome { + EcKvWriteOutcome::Written => Ok(()), + EcKvWriteOutcome::PreconditionFailed => Err(Report::new(match mode { + EcKvWriteMode::Add => KvWriteError::AlreadyExists, + _ => KvWriteError::Conflict, + })), + } + } + /// Creates a new entry, or overwrites an existing tombstone on re-consent. /// /// Three-way behavior: @@ -468,8 +467,9 @@ impl KvIdentityGraph { /// - **Existing tombstone** (`consent.ok = false`) — CAS overwrite with /// the new entry. Retries up to [`MAX_CAS_RETRIES`] on conflict. /// - /// This method is reserved for explicit same-key revival. Production EC - /// generation uses [`Self::create_if_absent`] with a freshly generated ID. + /// This method is reserved for explicit same-key revival. A freshly + /// generated EC's row is created by the navigation's page-load write + /// ([`Self::commit_page_load`]). /// /// # Errors /// @@ -555,159 +555,11 @@ impl KvIdentityGraph { ))) } - /// Persists browser EID cookies with one conditional write and one conflict read. - /// - /// Browser cookies do not carry a value-owned version, so a different - /// existing value has unknown freshness and is always deferred. After a CAS - /// conflict this method never writes again. A live follow-up becomes the - /// authoritative snapshot; a missing or failed follow-up retains the live - /// pre-write snapshot as proof and defers the update. - pub(crate) fn sync_eid_cookie_updates_from_snapshot( - &self, - ec_id: &str, - updates: &[PartnerIdUpdate], - snapshot: EcKvSnapshot, - ) -> (EcKvSnapshot, EidCookieSyncOutcome) { - if updates.is_empty() { - return (snapshot, EidCookieSyncOutcome::AlreadyMatched); - } - - let proven = match snapshot { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - .. - } if snapshot_id == ec_id => Some(snapshot.clone()), - _ => None, - }; - - let current = match snapshot { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - generation: Some(_), - .. - } if snapshot_id == ec_id => snapshot, - EcKvSnapshot::Failed { - ec_id: ref snapshot_id, - } if snapshot_id == ec_id => return (snapshot, EidCookieSyncOutcome::Failed), - _ => self.load_snapshot(ec_id), - }; - - let (mut entry, generation) = match current { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - ref entry, - generation: Some(generation), - } if snapshot_id == ec_id => (entry.as_ref().clone(), generation), - EcKvSnapshot::Missing { .. } => { - let outcome = if proven.is_some() { - EidCookieSyncOutcome::DeferredStaleRead - } else { - EidCookieSyncOutcome::Missing - }; - let kept = Self::keep_proven(ec_id, current, proven.as_ref()); - return (kept, outcome); - } - EcKvSnapshot::Failed { .. } => { - let kept = Self::keep_proven(ec_id, current, proven.as_ref()); - return (kept, EidCookieSyncOutcome::Failed); - } - EcKvSnapshot::Present { .. } | EcKvSnapshot::NotRead => { - return ( - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }, - EidCookieSyncOutcome::Failed, - ); - } - }; - - if !entry.consent.ok { - return (current, EidCookieSyncOutcome::ConsentWithdrawn); - } - let deferred_freshness = match apply_cookie_partner_id_updates(&mut entry, updates) { - CookieUpdateApplication::AlreadyMatched => { - return (current, EidCookieSyncOutcome::AlreadyMatched); - } - CookieUpdateApplication::DeferredFreshness => { - return (current, EidCookieSyncOutcome::DeferredFreshness); - } - CookieUpdateApplication::Changed { deferred } => deferred, - }; - - let Ok((body, meta_str)) = Self::serialize_entry(&entry, self.store_name()) else { - return ( - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }, - EidCookieSyncOutcome::Failed, - ); - }; - match self.write_entry( - ec_id, - &body, - &meta_str, - ENTRY_TTL, - EcKvWriteMode::IfGenerationMatch(generation), - ) { - Ok(EcKvWriteOutcome::Written) => ( - EcKvSnapshot::Present { - ec_id: ec_id.to_owned(), - entry: Box::new(entry), - generation: None, - }, - if deferred_freshness { - EidCookieSyncOutcome::WrittenWithDeferredFreshness - } else { - EidCookieSyncOutcome::Written - }, - ), - Ok(EcKvWriteOutcome::PreconditionFailed) => { - let refreshed = self.load_snapshot(ec_id); - let Some(refreshed_entry) = refreshed.entry_for(ec_id) else { - // The failed CAS proved `current`'s generation stale. Keep - // only its existence proof so later writes must reread. - let proof = match current { - EcKvSnapshot::Present { - ec_id: proven_id, - entry, - .. - } => EcKvSnapshot::Present { - ec_id: proven_id, - entry, - generation: None, - }, - other => other, - }; - let kept = Self::keep_proven(ec_id, refreshed, Some(&proof)); - return (kept, EidCookieSyncOutcome::DeferredConflict); - }; - if !refreshed_entry.consent.ok { - return (refreshed, EidCookieSyncOutcome::ConsentWithdrawn); - } - let outcome = if partner_id_updates_match(refreshed_entry, updates) { - EidCookieSyncOutcome::ConflictMatched - } else { - EidCookieSyncOutcome::DeferredConflict - }; - (refreshed, outcome) - } - Err(err) => { - log::warn!("EID cookie sync write failed: {err:?}"); - ( - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }, - EidCookieSyncOutcome::Failed, - ) - } - } - } - /// Merges partner IDs using request-scoped persisted state as the first CAS input. /// /// A caller-supplied `Present` snapshot is *proof* that the row exists — - /// either an `Add`-confirmed create from [`generate_if_needed`] or an - /// earlier authoritative read in the same request. Partner-ID enrichment is + /// an earlier authoritative read or a confirmed write in the same + /// request. Partner-ID enrichment is /// best effort, so a refresh that reads absent or unreadable never /// downgrades that proof: the update is skipped and logged, and the proven /// snapshot is returned so cookie issuance continues from the confirmed @@ -718,8 +570,6 @@ impl KvIdentityGraph { /// updates are dropped, because a point read on an eventually-consistent /// store cannot prove absence and routes without orphan recovery have no /// later chance to retry. - /// - /// [`generate_if_needed`]: super::generate_if_needed pub(crate) fn upsert_partner_ids_from_snapshot( &self, ec_id: &str, @@ -792,11 +642,29 @@ impl KvIdentityGraph { if !apply_partner_id_updates(&mut entry, updates) { return current; } - let Ok((body, meta_str)) = Self::serialize_entry(&entry, self.store_name()) else { - return EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), + let (body, meta_str) = + match Self::serialize_entry_unless_full(&entry, self.store_name()) { + Ok(SerializedEntry::Fits { body, meta }) => (body, meta), + Ok(SerializedEntry::TooLarge(bytes)) => { + log::warn!( + "snapshot partner upsert for '{}' skipped: row would be {bytes} bytes \ + (limit {MAX_ENTRY_BYTES})", + log_id(ec_id) + ); + return EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }; + } + Err(err) => { + log::warn!( + "snapshot partner upsert for '{}' could not serialize the row: {err:?}", + log_id(ec_id) + ); + return EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + }; + } }; - }; match self.write_entry( ec_id, &body, @@ -840,8 +708,8 @@ impl KvIdentityGraph { /// Returns `proven` instead of a read outcome that cannot disprove it. /// /// Partner-ID enrichment is best effort. When the caller already held proof - /// that the row exists — an `Add`-confirmed create or an earlier - /// authoritative read in the same request — a refresh that misses or fails + /// that the row exists — the page-load write's `Add`-confirmed create or an + /// earlier authoritative read in the same request — a refresh that misses or fails /// says nothing about existence, so the proof is kept and the skipped /// update is logged. Write failures are *not* routed here: they are real /// operation failures and stay [`EcKvSnapshot::Failed`] so callers can @@ -920,9 +788,7 @@ impl KvIdentityGraph { entry.ids.insert( partner_id.to_owned(), - super::kv_types::KvPartnerId { - uid: uid.to_owned(), - }, + super::kv_types::KvPartnerId::single(uid.to_owned()), ); let (body, meta_str) = Self::serialize_entry(&entry, self.store_name())?; @@ -992,12 +858,21 @@ impl KvIdentityGraph { entry.ids.insert( partner_id.to_owned(), - super::kv_types::KvPartnerId { - uid: uid.to_owned(), - }, + super::kv_types::KvPartnerId::single(uid.to_owned()), ); - let (body, meta_str) = Self::serialize_entry(&entry, self.store_name())?; + let (body, meta_str) = + match Self::serialize_entry_unless_full(&entry, self.store_name())? { + SerializedEntry::Fits { body, meta } => (body, meta), + SerializedEntry::TooLarge(bytes) => { + log::warn!( + "upsert_partner_id_if_exists: row for '{}' would be {bytes} bytes \ + (limit {MAX_ENTRY_BYTES}); partner '{partner_id}' not written", + log_id(ec_id), + ); + return Ok(UpsertResult::RowFull); + } + }; match self.write_entry( ec_id, @@ -1051,7 +926,12 @@ impl KvIdentityGraph { ) } - fn withdrawal_marker_exists( + /// Whether a still-valid withdrawal-completion marker exists for `ec_id`. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::KvStore`] when the marker listing fails. + pub(crate) fn withdrawal_marker_exists( &self, ec_id: &str, now: Option, @@ -1253,117 +1133,45 @@ impl KvIdentityGraph { }) } - /// Resolves a tombstone attempt whose point read reported the row absent. + /// Writes a withdrawal tombstone only when an existing row can be confirmed. /// - /// A completion marker proves an earlier withdrawal finished and avoids a - /// second root existence check. Otherwise, a proven-absent key is a no-op: - /// there is nothing to withdraw, and a forged cookie must not mint a row. A - /// key that provably exists is tombstoned unconditionally because no CAS - /// generation is available after a missed read. Marker-check failure falls - /// back to that privacy write, as does an unusable clock; root-existence - /// failure leaves withdrawal unresolved rather than silently dropped. - fn tombstone_unproven_missing( + /// This is a privacy write outside the one-write-per-page-load rule: it + /// runs on any finalized request that carries an explicit withdrawal and + /// an EC cookie, whatever the route or response status. + /// + /// Existing-key-only behavior is deliberate: a forged or expired `ts-ec` + /// cookie must not mint a row. But a *point read* cannot prove absence on + /// an eventually-consistent store, and dropping a withdrawal is worse than + /// a redundant read, so absence is established in two stages: + /// + /// 1. Any snapshot that is not a usable `Present` for this EC ID — a + /// publisher preload that read `Missing`, a read that `Failed`, or one + /// lacking a CAS generation — is re-read. + /// 2. A re-read that still reports the row absent is checked against + /// [`key_exists_confirmed`](Self::key_exists_confirmed), which reads + /// the primary data source. + /// + /// A tombstone is idempotent, so a CAS conflict re-reads and retries up to + /// [`MAX_CAS_RETRIES`] times. Resolving the initial snapshot happens + /// outside the retry counter. + pub(crate) fn tombstone_existing_from_snapshot( &self, ec_id: &str, - missing: EcKvSnapshot, - now: Option, + snapshot: EcKvSnapshot, ) -> EcKvSnapshot { - match self.withdrawal_marker_exists(ec_id, now) { - Ok(true) => { - log::debug!( - "withdrawal tombstone for '{}': completion marker already exists", - log_id(ec_id) - ); - return missing; - } - Ok(false) => {} - Err(err) => { - // Marker failure must not weaken withdrawal. Fall back to the - // existing root existence check and unconditional privacy write. - log::warn!( - "withdrawal completion marker lookup failed for '{}': {err:?}", - log_id(ec_id) - ); - } - } - - match self.key_exists_confirmed(ec_id) { - Ok(false) => missing, - Ok(true) => { - log::warn!( - "withdrawal tombstone for '{}': point read missed a row the store still \ - lists; writing an unconditional tombstone", - log_id(ec_id) - ); - match self.overwrite_withdrawal_tombstone(ec_id) { - Ok(tombstone) => { - self.record_withdrawal_completion(ec_id, tombstone.consent.updated); - EcKvSnapshot::Present { - ec_id: ec_id.to_owned(), - entry: Box::new(tombstone), - generation: None, - } - } - Err(err) => { - log::warn!( - "unconditional withdrawal tombstone failed for '{}': {err:?}", - log_id(ec_id) - ); - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - } - } - } - } - Err(err) => { - log::warn!( - "withdrawal tombstone for '{}': existence check failed, cannot confirm \ - absence: {err:?}", - log_id(ec_id) - ); - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - } - } - } - } - - /// Writes a tombstone only when an existing row can be confirmed. - /// - /// Existing-key-only behavior is deliberate: a forged or expired `ts-ec` - /// cookie must not mint a row. But a *point read* cannot prove absence on - /// an eventually-consistent store, and dropping a withdrawal is worse than - /// a redundant read, so absence is established in two stages: - /// - /// 1. Any snapshot that is not a usable `Present` for this EC ID — a - /// publisher preload that read `Missing`, a read that `Failed`, or one - /// lacking a CAS generation — is re-read. On the publisher path that - /// re-read is separated from the preload by the full origin round trip, - /// which gives replication time to converge. - /// 2. A re-read that still reports the row absent is checked against - /// [`key_exists_confirmed`](Self::key_exists_confirmed), which reads - /// the primary data source. - /// - /// Resolving the initial snapshot happens outside the retry counter, so all - /// [`MAX_CAS_RETRIES`] iterations stay available for the tombstone write. - pub(crate) fn tombstone_existing_from_snapshot( - &self, - ec_id: &str, - snapshot: EcKvSnapshot, - ) -> EcKvSnapshot { - let mut current = match snapshot { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - ref entry, - .. - } if snapshot_id == ec_id && !entry.consent.ok => return snapshot, - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - generation: Some(_), - .. - } if snapshot_id == ec_id => snapshot, - _ => self.load_snapshot(ec_id), - }; + let mut current = match snapshot { + EcKvSnapshot::Present { + ec_id: ref snapshot_id, + ref entry, + .. + } if snapshot_id == ec_id && !entry.consent.ok => return snapshot, + EcKvSnapshot::Present { + ec_id: ref snapshot_id, + generation: Some(_), + .. + } if snapshot_id == ec_id => snapshot, + _ => self.load_snapshot(ec_id), + }; for _attempt in 0..MAX_CAS_RETRIES { let generation = match current { @@ -1434,9 +1242,6 @@ impl KvIdentityGraph { } } - // Withdrawal enforcement lost every CAS race, so the row can still be - // live with consent granted while the browser cookie is cleared. That - // divergence is only visible to operators if it is logged here. log::warn!( "withdrawal tombstone for '{}': CAS conflict after {MAX_CAS_RETRIES} retries; the \ identity-graph row may still be live with consent granted", @@ -1447,6 +1252,80 @@ impl KvIdentityGraph { } } + /// Resolves a withdrawal whose row read as missing. + /// + /// A still-valid completion marker or a confirmed absent root means + /// there is nothing to withdraw, and a forged cookie must not mint a row. A + /// key that provably exists is tombstoned unconditionally because no CAS + /// generation is available after a missed read. Marker-check failure falls + /// back to that privacy write; root-existence failure leaves withdrawal + /// unresolved rather than silently dropped. + fn tombstone_unproven_missing( + &self, + ec_id: &str, + missing: EcKvSnapshot, + now: Option, + ) -> EcKvSnapshot { + match self.withdrawal_marker_exists(ec_id, now) { + Ok(true) => { + log::debug!( + "withdrawal tombstone for '{}': completion marker already exists", + log_id(ec_id) + ); + return missing; + } + Ok(false) => {} + Err(err) => { + // Marker failure must not weaken withdrawal. Fall back to the + // existing root existence check and unconditional privacy write. + log::warn!( + "withdrawal completion marker lookup failed for '{}': {err:?}", + log_id(ec_id) + ); + } + } + + match self.key_exists_confirmed(ec_id) { + Ok(false) => missing, + Ok(true) => { + log::warn!( + "withdrawal tombstone for '{}': point read missed a row the store still \ + lists; writing an unconditional tombstone", + log_id(ec_id) + ); + match self.overwrite_withdrawal_tombstone(ec_id) { + Ok(tombstone) => { + self.record_withdrawal_completion(ec_id, tombstone.consent.updated); + EcKvSnapshot::Present { + ec_id: ec_id.to_owned(), + entry: Box::new(tombstone), + generation: None, + } + } + Err(err) => { + log::warn!( + "unconditional withdrawal tombstone failed for '{}': {err:?}", + log_id(ec_id) + ); + EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + } + } + } + } + Err(err) => { + log::warn!( + "withdrawal tombstone for '{}': existence check failed, cannot confirm \ + absence: {err:?}", + log_id(ec_id) + ); + EcKvSnapshot::Failed { + ec_id: ec_id.to_owned(), + } + } + } + } + /// Counts the number of keys sharing the same EC hash prefix. /// /// Uses the platform KV list API with a prefix filter, limited to @@ -1519,7 +1398,20 @@ impl KvIdentityGraph { network.cluster_size = Some(cluster_size); updated_entry.network = Some(network); - let (body, meta_str) = Self::serialize_entry(&updated_entry, self.store_name())?; + let (body, meta_str) = + match Self::serialize_entry_unless_full(&updated_entry, self.store_name())? { + SerializedEntry::Fits { body, meta } => (body, meta), + SerializedEntry::TooLarge(bytes) => { + // The computed value is still valid; only the cache write + // is skipped. + log::warn!( + "evaluate_cluster: row for '{}' would be {bytes} bytes \ + (limit {MAX_ENTRY_BYTES}); cluster_size not stored", + log_id(ec_id) + ); + return Ok(Some(cluster_size)); + } + }; match self.write_entry( ec_id, @@ -1551,7 +1443,8 @@ impl KvIdentityGraph { /// Hard-deletes the entry and any withdrawal completion marker. /// /// Reserved for the IAB data deletion framework (deferred). Consent - /// withdrawal uses [`Self::tombstone_existing_from_snapshot`] instead. + /// withdrawal writes a tombstone through `tombstone_existing_from_snapshot` + /// instead. /// /// # Errors /// @@ -1574,6 +1467,16 @@ impl KvIdentityGraph { )) } + /// A graph holding `entry` at `ec_id` whose reads succeed and whose + /// writes all fail. + pub(crate) fn write_failing_with(ec_id: &str, entry: &KvEntry) -> Self { + let store = super::kv_backend::test_support::WriteFailingEcKv::new(); + let (body, meta) = Self::serialize_entry(entry, store.store_name()) + .expect("should serialize the seed row"); + store.seed(ec_id, &body, &meta); + Self::new(store) + } + /// Test helper: a graph backed by an in-memory store with generation /// tracking. pub(crate) fn in_memory(store_name: impl Into) -> Self { @@ -1618,86 +1521,10 @@ impl KvIdentityGraph { #[cfg(test)] mod tests { use super::*; - use crate::ec::kv_backend::test_support::InMemoryEcKv; - - /// [`EcKvStore`] wrapper whose first CAS write both fails the precondition - /// and deletes the key, simulating a concurrent withdrawal that removes the - /// row between this writer's read and its write. - struct DisappearOnConflictEcKv { - inner: InMemoryEcKv, - conflicts_remaining: std::sync::Mutex, - } - - impl DisappearOnConflictEcKv { - fn new(conflicts: u32) -> Self { - Self { - inner: InMemoryEcKv::new("disappear-store"), - conflicts_remaining: std::sync::Mutex::new(conflicts), - } - } - - fn seed_live(&self, ec_id: &str) { - let (body, meta) = - KvIdentityGraph::serialize_entry(&live_entry(), self.inner.store_name()) - .expect("should serialize seeded entry"); - self.inner - .insert( - ec_id, - EcKvWrite { - body: &body, - metadata: &meta, - ttl: ENTRY_TTL, - mode: EcKvWriteMode::Add, - }, - ) - .expect("should seed live entry"); - } - } - - impl EcKvStore for DisappearOnConflictEcKv { - fn store_name(&self) -> &str { - self.inner.store_name() - } - - fn lookup(&self, key: &str) -> Result, Report> { - self.inner.lookup(key) - } - - fn key_exists(&self, key: &str) -> Result> { - self.inner.key_exists(key) - } - - fn insert( - &self, - key: &str, - write: EcKvWrite<'_>, - ) -> Result> { - if matches!(write.mode, EcKvWriteMode::IfGenerationMatch(_)) { - let mut remaining = self - .conflicts_remaining - .lock() - .expect("should lock conflict counter"); - if *remaining > 0 { - *remaining -= 1; - self.inner.delete(key).expect("should delete on conflict"); - return Ok(EcKvWriteOutcome::PreconditionFailed); - } - } - self.inner.insert(key, write) - } - - fn list_keys_with_prefix( - &self, - prefix: &str, - limit: u32, - ) -> Result, Report> { - self.inner.list_keys_with_prefix(prefix, limit) - } - - fn delete(&self, key: &str) -> Result<(), Report> { - self.inner.delete(key) - } - } + use crate::ec::checked_current_timestamp; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, UidValue, UnixSeconds}; + use crate::ec::kv_backend::test_support::{InMemoryEcKv, WriteFailingEcKv}; + use crate::ec::kv_types::{KvEid, KvUid}; fn snapshot_ec_id() -> String { format!("{}.ABC123", "a".repeat(64)) @@ -1733,22 +1560,27 @@ mod tests { } #[test] - fn deserialize_entry_rejects_invalid_legacy_values() { + fn deserialize_entry_drops_oversized_ids() { let mut entry = KvEntry::tombstone(1000); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1), - }, + crate::ec::kv_types::KvPartnerId::single( + "x".repeat(crate::ec::kv_types::MAX_STORED_UID_LENGTH + 1), + ), + ); + entry.ids.insert( + "ssp_y".to_owned(), + crate::ec::kv_types::KvPartnerId::single("kept"), ); - let body = serde_json::to_vec(&entry).expect("should serialize invalid entry payload"); + let body = serde_json::to_vec(&entry).expect("should serialize oversized entry payload"); - let err = KvIdentityGraph::deserialize_entry("test-store", "ec-id", &body) - .expect_err("should reject invalid legacy entry values"); - let err_text = format!("{err}"); - assert!( - err_text.contains("Loaded invalid entry"), - "should report validation failure for loaded entries" + let loaded = KvIdentityGraph::deserialize_entry("test-store", "ec-id", &body) + .expect("should load an entry holding an oversized ID"); + + assert_eq!( + loaded.ids.keys().collect::>(), + vec!["ssp_y"], + "should drop only the oversized ID" ); } @@ -1772,9 +1604,9 @@ mod tests { let mut entry = KvEntry::tombstone(1000); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1), - }, + crate::ec::kv_types::KvPartnerId::single( + "x".repeat(crate::ec::kv_types::MAX_STORED_UID_LENGTH + 1), + ), ); let err = KvIdentityGraph::serialize_entry(&entry, "test-store") @@ -1786,19 +1618,59 @@ mod tests { ); } - fn live_entry() -> KvEntry { - let mut entry = KvEntry::tombstone(1000); - entry.consent.ok = true; - entry - } - - fn concurrent_live_entry() -> KvEntry { + #[test] + fn create_refuses_oversized_provider_entry_before_write() { + let graph = KvIdentityGraph::in_memory("test-store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let uids = (0..40) + .map(|index| { + let id = format!("{index:03}{}", "x".repeat(480)); + KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::Never, + ) + }) + .collect(); + let record = crate::ec::kv_types::KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), uids)], + UnixSeconds::new(0), + ) + .expect("should build record"); let mut entry = live_entry(); - entry.ids.insert( - "concurrent.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "concurrent-uid".to_owned(), - }, + entry.ids.insert("ids.example.com".to_owned(), record); + assert!( + entry.validate().is_ok(), + "should be a valid entry apart from its size" + ); + + let err = graph + .create(&ec_id, &entry) + .expect_err("should refuse oversized entry"); + + assert!( + format!("{err}").contains("Refusing to write KV entry"), + "should report the size limit" + ); + assert!( + graph.get(&ec_id).expect("should read store").is_none(), + "should not write the oversized entry" + ); + } + + fn live_entry() -> KvEntry { + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry + } + + fn concurrent_live_entry() -> KvEntry { + let mut entry = live_entry(); + entry.ids.insert( + "concurrent.example.com".to_owned(), + crate::ec::kv_types::KvPartnerId::single("concurrent-uid".to_owned()), ); entry } @@ -1828,15 +1700,6 @@ mod tests { } } - fn with_partner_update_on_conflict(conflicts: u32) -> Self { - Self { - inner: InMemoryEcKv::new("partner-conflict-store"), - conflicts_remaining: std::sync::Mutex::new(conflicts), - revive_on_conflict: true, - partner_update_on_conflict: true, - } - } - fn seed_tombstone(&self, ec_id: &str) { let (body, meta) = KvIdentityGraph::serialize_entry( &KvEntry::tombstone(1000), @@ -1943,131 +1806,6 @@ mod tests { } } - /// Store that replaces the row during the first EID CAS write and records - /// the request's reads and conditional writes. - struct EidConflictEcKv { - inner: InMemoryEcKv, - concurrent_entry: KvEntry, - lookups: std::sync::Arc, - conditional_writes: std::sync::Arc, - follow_up_miss: bool, - miss_next_lookup: std::sync::atomic::AtomicBool, - } - - impl EidConflictEcKv { - fn new( - concurrent_entry: KvEntry, - lookups: std::sync::Arc, - conditional_writes: std::sync::Arc, - ) -> Self { - Self { - inner: InMemoryEcKv::new("eid-conflict-store"), - concurrent_entry, - lookups, - conditional_writes, - follow_up_miss: false, - miss_next_lookup: std::sync::atomic::AtomicBool::new(false), - } - } - - fn with_follow_up_miss(mut self) -> Self { - self.follow_up_miss = true; - self - } - - fn seed_live(&self, ec_id: &str) { - let (body, meta) = - KvIdentityGraph::serialize_entry(&live_entry(), self.inner.store_name()) - .expect("should serialize initial entry"); - self.inner - .insert( - ec_id, - EcKvWrite { - body: &body, - metadata: &meta, - ttl: ENTRY_TTL, - mode: EcKvWriteMode::Add, - }, - ) - .expect("should seed initial entry"); - } - } - - impl EcKvStore for EidConflictEcKv { - fn store_name(&self) -> &str { - self.inner.store_name() - } - - fn lookup(&self, key: &str) -> Result, Report> { - self.lookups - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - if self - .miss_next_lookup - .swap(false, std::sync::atomic::Ordering::Relaxed) - { - return Ok(None); - } - self.inner.lookup(key) - } - - fn key_exists(&self, key: &str) -> Result> { - self.inner.key_exists(key) - } - - fn insert( - &self, - key: &str, - write: EcKvWrite<'_>, - ) -> Result> { - if matches!(write.mode, EcKvWriteMode::IfGenerationMatch(_)) { - self.conditional_writes - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let (body, meta) = KvIdentityGraph::serialize_entry( - &self.concurrent_entry, - self.inner.store_name(), - ) - .expect("should serialize concurrent entry"); - self.inner - .insert( - key, - EcKvWrite { - body: &body, - metadata: &meta, - ttl: ENTRY_TTL, - mode: EcKvWriteMode::Overwrite, - }, - ) - .expect("should write concurrent entry"); - if self.follow_up_miss { - self.miss_next_lookup - .store(true, std::sync::atomic::Ordering::Relaxed); - } - return Ok(EcKvWriteOutcome::PreconditionFailed); - } - self.inner.insert(key, write) - } - - fn list_keys_with_prefix( - &self, - prefix: &str, - limit: u32, - ) -> Result, Report> { - self.inner.list_keys_with_prefix(prefix, limit) - } - - fn count_keys_with_prefix( - &self, - prefix: &str, - limit: u32, - ) -> Result> { - self.inner.count_keys_with_prefix(prefix, limit) - } - - fn delete(&self, key: &str) -> Result<(), Report> { - self.inner.delete(key) - } - } - #[test] fn create_or_revive_retries_cas_conflict_and_succeeds() { let store = ConflictInjectingEcKv::new(2, false); @@ -2131,9 +1869,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "uid-1".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("uid-1".to_owned()), ); let updates = vec![PartnerIdUpdate::new("ssp_x", "uid-1")]; @@ -2159,9 +1895,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "old-uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("old-uid".to_owned()), ); let updates = vec![PartnerIdUpdate::new("ssp_x", "new-uid")]; @@ -2191,9 +1925,7 @@ mod tests { let mut entry = live_entry(); entry.ids.insert( "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "original".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("original".to_owned()), ); let updates = vec![ PartnerIdUpdate::new("ssp_x", "intermediate"), @@ -2210,734 +1942,623 @@ mod tests { } #[test] - fn eid_cookie_sync_conflict_matching_value_stops_after_one_write() { - let ec_id = snapshot_ec_id(); - let mut concurrent = live_entry(); - concurrent.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "desired-uid".to_owned(), - }, - ); - let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let store = EidConflictEcKv::new( - concurrent, - std::sync::Arc::clone(&lookups), - std::sync::Arc::clone(&writes), - ); - store.seed_live(&ec_id); - let graph = KvIdentityGraph::new(store); - let snapshot = graph.load_snapshot(&ec_id); + fn evaluate_cluster_returns_stored_value_without_store_io() { + let kv = KvIdentityGraph::failing("nonexistent_store_for_cluster_cache_test"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let mut entry = live_entry(); + entry.network = Some(KvNetwork { + cluster_size: Some(5), + }); - let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "desired-uid")], - snapshot, - ); + let cluster_size = kv + .evaluate_cluster(&ec_id, &entry, 0) + .expect("should not touch store when cluster_size is already known"); - assert_eq!(outcome, EidCookieSyncOutcome::ConflictMatched); - assert_eq!( - snapshot - .entry_for(&ec_id) - .and_then(|entry| entry.ids.get("ssp_x")) - .map(|id| id.uid.as_str()), - Some("desired-uid"), - "the authoritative follow-up snapshot should replace stale request state" - ); - assert_eq!( - lookups.load(std::sync::atomic::Ordering::Relaxed), - 2, - "a conflict should perform exactly one follow-up read" - ); assert_eq!( - writes.load(std::sync::atomic::Ordering::Relaxed), - 1, - "a conflict must not trigger another conditional write" + cluster_size, + Some(5), + "should return stored cluster_size without re-listing keys" ); } #[test] - fn eid_cookie_sync_keeps_live_proof_when_conflict_follow_up_misses() { - let ec_id = snapshot_ec_id(); - let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let store = EidConflictEcKv::new( - live_entry(), - std::sync::Arc::clone(&lookups), - std::sync::Arc::clone(&writes), - ) - .with_follow_up_miss(); - store.seed_live(&ec_id); - let graph = KvIdentityGraph::new(store); + fn evaluate_cluster_skips_tombstone_without_store_io() { + let kv = KvIdentityGraph::failing("nonexistent_store_for_tombstone_cluster_test"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let entry = KvEntry::tombstone(1000); - let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "desired-uid")], - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); + let cluster_size = kv + .evaluate_cluster(&ec_id, &entry, 0) + .expect("should not touch store for tombstone entries"); - assert_eq!(outcome, EidCookieSyncOutcome::DeferredConflict); - assert!( - snapshot.entry_for(&ec_id).is_some(), - "the live pre-write row should prevent conflict deferral from entering orphan recovery" - ); - assert_eq!( - snapshot.generation_for(&ec_id), - None, - "a failed CAS must not return its rejected generation" - ); - assert_eq!( - lookups.load(std::sync::atomic::Ordering::Relaxed), - 2, - "the initial refresh and conflict follow-up should be the only reads" - ); assert_eq!( - writes.load(std::sync::atomic::Ordering::Relaxed), - 1, - "the conflict must remain the request's only conditional write" + cluster_size, None, + "should not evaluate or write cluster_size for tombstones" ); } #[test] - fn eid_cookie_sync_reports_proven_refresh_miss_as_deferred() { - let ec_id = snapshot_ec_id(); - let graph = KvIdentityGraph::in_memory("empty-store"); - let proven = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: None, - }; + fn create_then_get_roundtrips_entry() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let entry = live_entry(); - let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "desired-uid")], - proven, - ); + kv.create(&ec_id, &entry).expect("should create new entry"); + let (loaded, generation) = kv + .get(&ec_id) + .expect("should read entry back") + .expect("should find created entry"); - assert!( - snapshot.entry_for(&ec_id).is_some(), - "the add-confirmed row should survive an eventually consistent miss" - ); - assert_eq!( - outcome, - EidCookieSyncOutcome::DeferredStaleRead, - "a row already proven to exist should report a deferred stale read" - ); + assert!(loaded.consent.ok, "should preserve consent state"); + assert!(generation > 0, "should expose a generation marker"); } #[test] - fn partner_id_conflict_match_uses_last_duplicate_value() { - let mut entry = live_entry(); - entry.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "latest-uid".to_owned(), - }, - ); - let updates = [ - PartnerIdUpdate::new("ssp_x", "older-uid"), - PartnerIdUpdate::new("ssp_x", "latest-uid"), - ]; + fn create_rejects_existing_key() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let entry = live_entry(); + kv.create(&ec_id, &entry).expect("should create new entry"); + let err = kv + .create(&ec_id, &entry) + .expect_err("should reject duplicate create"); assert!( - partner_id_updates_match(&entry, &updates), - "conflict matching should use the same last-value-wins rule as writes" + format!("{err}").contains("already exists"), + "should report duplicate key" ); } #[test] - fn eid_cookie_sync_conflicting_value_defers_after_one_write() { - let ec_id = snapshot_ec_id(); - let mut concurrent = live_entry(); - concurrent.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "concurrent-uid".to_owned(), - }, - ); - let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let store = EidConflictEcKv::new( - concurrent, - std::sync::Arc::clone(&lookups), - std::sync::Arc::clone(&writes), - ); - store.seed_live(&ec_id); - let graph = KvIdentityGraph::new(store); - let snapshot = graph.load_snapshot(&ec_id); + fn create_or_revive_revives_tombstone() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); - let (_, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "stale-uid")], - snapshot, - ); + kv.create(&ec_id, &KvEntry::tombstone(1000)) + .expect("should create tombstone"); + kv.create_or_revive(&ec_id, &live_entry()) + .expect("should revive tombstone"); - assert_eq!(outcome, EidCookieSyncOutcome::DeferredConflict); - assert_eq!( - lookups.load(std::sync::atomic::Ordering::Relaxed), - 2, - "a conflict should perform exactly one follow-up read" - ); - assert_eq!( - writes.load(std::sync::atomic::Ordering::Relaxed), - 1, - "a conflict must not trigger another conditional write" - ); - let (stored, _) = graph + let (loaded, _) = kv .get(&ec_id) - .expect("should read concurrent row") - .expect("row should remain"); - assert_eq!(stored.ids["ssp_x"].uid, "concurrent-uid"); + .expect("should read entry back") + .expect("should find revived entry"); + assert!(loaded.consent.ok, "should be live after revive"); } #[test] - fn eid_cookie_sync_preserves_concurrent_withdrawal_after_conflict() { - let ec_id = snapshot_ec_id(); - let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let writes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let store = EidConflictEcKv::new( - KvEntry::tombstone(2_000), - std::sync::Arc::clone(&lookups), - std::sync::Arc::clone(&writes), - ); - store.seed_live(&ec_id); - let graph = KvIdentityGraph::new(store); - let snapshot = graph.load_snapshot(&ec_id); + fn create_or_revive_fresh_entry_does_not_access_marker_store() { + let operations = Arc::new(RecordedEcKvOperations::default()); + let kv = KvIdentityGraph::new(RecordingEcKv::with_marker_failures( + Arc::clone(&operations), + 0, + )); + let ec_id = format!("{}.ABC123", "a".repeat(64)); - let (snapshot, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "desired-uid")], - snapshot, - ); + kv.create_or_revive(&ec_id, &live_entry()) + .expect("should create without reading withdrawal markers"); - assert_eq!(outcome, EidCookieSyncOutcome::ConsentWithdrawn); - assert!( - snapshot - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "the refreshed withdrawal tombstone must remain authoritative" - ); assert_eq!( - lookups.load(std::sync::atomic::Ordering::Relaxed), - 2, - "a conflict should perform exactly one follow-up read" + operations.list_count(), + 0, + "fresh create should not list withdrawal markers" ); assert_eq!( - writes.load(std::sync::atomic::Ordering::Relaxed), + operations.inserts().len(), 1, - "a conflict must not trigger another conditional write" + "fresh create should only insert the live root" + ); + assert!( + kv.get(&ec_id) + .expect("should read entry") + .is_some_and(|(entry, _)| entry.consent.ok), + "fresh create should persist a live entry" ); } #[test] - fn eid_cookie_sync_defers_different_values_without_value_owned_freshness() { + fn withdrawal_marker_validity_requires_a_usable_clock_before_expiry() { + let graph = KvIdentityGraph::in_memory("test_store"); let ec_id = snapshot_ec_id(); - let graph = KvIdentityGraph::in_memory("freshness-store"); - let mut entry = live_entry(); - entry.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "protected-uid".to_owned(), - }, - ); - graph.create(&ec_id, &entry).expect("should seed entry"); - - let (_, outcome) = graph.sync_eid_cookie_updates_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "incoming-uid")], - graph.load_snapshot(&ec_id), - ); + graph + .write_withdrawal_marker(&ec_id, 1000) + .expect("should seed marker"); + let valid_until = 1000 + TOMBSTONE_TTL.as_secs(); - assert_eq!(outcome, EidCookieSyncOutcome::DeferredFreshness); - let (stored, _) = graph - .get(&ec_id) - .expect("should read protected row") - .expect("row should remain"); - assert_eq!(stored.ids["ssp_x"].uid, "protected-uid"); + for (now, expected) in [ + (Some(valid_until - 1), true), + (Some(valid_until), false), + (Some(valid_until + 1), false), + (None, false), + ] { + assert_eq!( + graph + .withdrawal_marker_exists(&ec_id, now) + .expect("should check marker validity"), + expected, + "marker validity should honor clock availability and exclusive expiry at {now:?}" + ); + } } #[test] - fn eid_cookie_sync_adds_missing_ids_while_deferring_different_values() { - let ec_id = snapshot_ec_id(); - let graph = KvIdentityGraph::in_memory("mixed-freshness-store"); - let mut entry = live_entry(); - entry.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "protected-uid".to_owned(), - }, - ); - graph.create(&ec_id, &entry).expect("should seed entry"); - - let (_, outcome) = graph.sync_eid_cookie_updates_from_snapshot( + fn withdrawal_marker_existence_rejects_malformed_expiry_suffix() { + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let marker_key = KvIdentityGraph::withdrawal_marker_key( &ec_id, - &[ - PartnerIdUpdate::new("ssp_x", "different-uid"), - PartnerIdUpdate::new("ssp_y", "new-uid"), - ], - graph.load_snapshot(&ec_id), + current_timestamp() + TOMBSTONE_TTL.as_secs(), ); + let longer_key = format!("{marker_key}-longer"); + let store = InMemoryEcKv::new("test_store"); + store + .insert( + &longer_key, + EcKvWrite { + body: "1", + metadata: "{}", + ttl: TOMBSTONE_TTL, + mode: EcKvWriteMode::Add, + }, + ) + .expect("should seed longer marker key"); + let kv = KvIdentityGraph::new(store); - assert_eq!(outcome, EidCookieSyncOutcome::WrittenWithDeferredFreshness); - let (stored, _) = graph - .get(&ec_id) - .expect("should read updated row") - .expect("row should remain"); - assert_eq!(stored.ids["ssp_x"].uid, "protected-uid"); - assert_eq!(stored.ids["ssp_y"].uid, "new-uid"); + assert!( + !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) + .expect("should validate withdrawal marker expiry"), + "a malformed expiry suffix must not prove withdrawal completion" + ); } #[test] - fn evaluate_cluster_returns_stored_value_without_store_io() { - let kv = KvIdentityGraph::failing("nonexistent_store_for_cluster_cache_test"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let mut entry = live_entry(); - entry.network = Some(KvNetwork { - cluster_size: Some(5), - }); + fn clear_absent_withdrawal_marker_lists_once_without_delete() { + let operations = Arc::new(RecordedEcKvOperations::default()); + let kv = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); + let ec_id = snapshot_ec_id(); - let cluster_size = kv - .evaluate_cluster(&ec_id, &entry, 0) - .expect("should not touch store when cluster_size is already known"); + kv.clear_withdrawal_marker(&ec_id) + .expect("should accept an absent marker"); assert_eq!( - cluster_size, - Some(5), - "should return stored cluster_size without re-listing keys" + operations.list_count(), + 1, + "absent marker guard should list once" + ); + assert_eq!( + operations.delete_count(), + 0, + "absent marker guard should avoid a delete" ); } #[test] - fn evaluate_cluster_skips_tombstone_without_store_io() { - let kv = KvIdentityGraph::failing("nonexistent_store_for_tombstone_cluster_test"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let entry = KvEntry::tombstone(1000); + fn writing_an_existing_withdrawal_marker_is_idempotent() { + let operations = Arc::new(RecordedEcKvOperations::default()); + let kv = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); + let ec_id = snapshot_ec_id(); + let updated = current_timestamp(); - let cluster_size = kv - .evaluate_cluster(&ec_id, &entry, 0) - .expect("should not touch store for tombstone entries"); + kv.write_withdrawal_marker(&ec_id, updated) + .expect("should write first completion marker"); + kv.write_withdrawal_marker(&ec_id, updated) + .expect("should accept completion marker add collision"); assert_eq!( - cluster_size, None, - "should not evaluate or write cluster_size for tombstones" + operations.inserts(), + vec![ + RecordedEcKvInsert { + mode: EcKvWriteMode::Add, + ttl: TOMBSTONE_TTL, + }, + RecordedEcKvInsert { + mode: EcKvWriteMode::Add, + ttl: TOMBSTONE_TTL, + }, + ], + "both marker add attempts should reach the store" + ); + assert!( + kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) + .expect("should read completion marker"), + "completion marker should remain valid" ); } #[test] - fn create_then_get_roundtrips_entry() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let entry = live_entry(); + fn marker_list_saturation_blocks_revival_and_hard_delete_before_mutation() { + let operations = Arc::new(RecordedEcKvOperations::default()); + let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); + let ec_id = snapshot_ec_id(); + graph + .create(&ec_id, &KvEntry::tombstone(current_timestamp())) + .expect("should seed tombstone"); + let first_valid_until = current_timestamp() + TOMBSTONE_TTL.as_secs(); + for offset in 0..WITHDRAWAL_MARKER_LIST_LIMIT { + let marker_key = KvIdentityGraph::withdrawal_marker_key( + &ec_id, + first_valid_until + u64::from(offset), + ); + graph + .store + .insert( + &marker_key, + EcKvWrite { + body: "1", + metadata: "{}", + ttl: TOMBSTONE_TTL, + mode: EcKvWriteMode::Add, + }, + ) + .expect("should seed completion marker"); + } + operations.reset(); - kv.create(&ec_id, &entry).expect("should create new entry"); - let (loaded, generation) = kv - .get(&ec_id) - .expect("should read entry back") - .expect("should find created entry"); + let result = graph.create_or_revive(&ec_id, &live_entry()); - assert!(loaded.consent.ok, "should preserve consent state"); - assert!(generation > 0, "should expose a generation marker"); - } + assert!( + result.is_err(), + "revival should fail when marker cleanup cannot prove the prefix is exhausted" + ); + let (stored, _) = graph + .get(&ec_id) + .expect("should read root") + .expect("should retain root"); + assert!(!stored.consent.ok, "root should remain tombstoned"); + assert_eq!( + operations.delete_count(), + 0, + "saturated marker cleanup should not make partial progress" + ); - #[test] - fn create_rejects_existing_key() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let entry = live_entry(); + operations.reset(); + let delete_result = graph.delete(&ec_id); - kv.create(&ec_id, &entry).expect("should create new entry"); - let err = kv - .create(&ec_id, &entry) - .expect_err("should reject duplicate create"); assert!( - format!("{err}").contains("already exists"), - "should report duplicate key" + delete_result.is_err(), + "hard deletion should fail when marker cleanup cannot prove the prefix is exhausted" + ); + let (stored, _) = graph + .get(&ec_id) + .expect("should read root after rejected hard delete") + .expect("should retain the root after a rejected hard delete"); + assert!( + !stored.consent.ok, + "rejected hard delete should leave the tombstoned root in place" + ); + assert_eq!( + operations.delete_count(), + 0, + "hard deletion should not remove the root before saturated marker cleanup fails" ); } #[test] - fn create_if_absent_reports_written_and_collision() { + fn upsert_partner_id_if_exists_reports_missing_key() { let kv = KvIdentityGraph::in_memory("test_store"); let ec_id = format!("{}.ABC123", "a".repeat(64)); + let result = kv + .upsert_partner_id_if_exists(&ec_id, "ssp_x", "uid-1") + .expect("should not error on missing key"); assert_eq!( - kv.create_if_absent(&ec_id, &live_entry()) - .expect("should create absent entry"), - CreateIfAbsentOutcome::Written - ); - assert_eq!( - kv.create_if_absent(&ec_id, &live_entry()) - .expect("should report collision"), - CreateIfAbsentOutcome::AlreadyExists + result, + UpsertResult::NotFound, + "should reject a partner upsert for an EC ID the store does not hold" ); } #[test] - fn create_if_absent_propagates_store_error() { - let kv = KvIdentityGraph::failing("test_store"); + fn upsert_partner_id_if_exists_reports_a_full_row_without_writing() { + let kv = KvIdentityGraph::in_memory("test_store"); let ec_id = format!("{}.ABC123", "a".repeat(64)); + let mut entry = live_entry(); + let uid = "u".repeat(500); + let mut partner = 0; + while serde_json::to_string(&entry) + .expect("should serialize entry") + .len() + < MAX_ENTRY_BYTES - 600 + { + entry.ids.insert( + format!("ssp{partner}.example.com"), + crate::ec::kv_types::KvPartnerId::single(uid.clone()), + ); + partner += 1; + } + kv.create(&ec_id, &entry) + .expect("should create a near-full row"); + + let result = kv + .upsert_partner_id_if_exists(&ec_id, "full.example.com", &uid) + .expect("should report a full row as a result, not an error"); + assert_eq!( + result, + UpsertResult::RowFull, + "should refuse a partner ID that would overflow the row" + ); + let (stored, _) = kv + .get(&ec_id) + .expect("should read the row") + .expect("should keep the row"); assert!( - kv.create_if_absent(&ec_id, &live_entry()).is_err(), - "should preserve store failures instead of reporting a collision" + !stored.ids.contains_key("full.example.com"), + "should not write the refused partner ID" ); } #[test] - fn create_or_revive_revives_tombstone() { + fn upsert_partner_id_if_exists_writes_and_detects_unchanged() { let kv = KvIdentityGraph::in_memory("test_store"); let ec_id = format!("{}.ABC123", "a".repeat(64)); + kv.create(&ec_id, &live_entry()).expect("should create"); - kv.create(&ec_id, &KvEntry::tombstone(1000)) - .expect("should create tombstone"); - kv.create_or_revive(&ec_id, &live_entry()) - .expect("should revive tombstone"); - - let (loaded, _) = kv - .get(&ec_id) - .expect("should read entry back") - .expect("should find revived entry"); - assert!(loaded.consent.ok, "should be live after revive"); + let first = kv + .upsert_partner_id_if_exists(&ec_id, "ssp_x", "uid-1") + .expect("should write partner id"); + assert_eq!( + first, + UpsertResult::Written, + "should report the first partner upsert as written" + ); + + let second = kv + .upsert_partner_id_if_exists(&ec_id, "ssp_x", "uid-1") + .expect("should detect unchanged uid"); + assert_eq!( + second, + UpsertResult::Unchanged, + "should report an identical partner upsert as unchanged" + ); } #[test] - fn create_or_revive_fresh_entry_does_not_access_marker_store() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let kv = KvIdentityGraph::new(RecordingEcKv::with_marker_failures( - Arc::clone(&operations), - 0, - )); + fn upsert_partner_id_if_exists_retries_cas_conflict() { + let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(1, false)); let ec_id = format!("{}.ABC123", "a".repeat(64)); + graph + .create(&ec_id, &live_entry()) + .expect("should seed live entry"); - kv.create_or_revive(&ec_id, &live_entry()) - .expect("should create without reading withdrawal markers"); + let result = graph + .upsert_partner_id_if_exists(&ec_id, "ssp_x", "uid-1") + .expect("should retry and write after one generation conflict"); + assert_eq!(result, UpsertResult::Written); + let (entry, _) = graph + .get(&ec_id) + .expect("should read persisted entry") + .expect("should retain entry after conflict"); assert_eq!( - operations.list_count(), - 0, - "fresh create should not list withdrawal markers" + entry.ids.get("ssp_x").map(|id| id.uid.as_str()), + Some("uid-1"), + "should persist requested UID after retry" ); + } + + #[test] + fn upsert_partner_id_if_exists_rejects_tombstone() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + kv.create(&ec_id, &KvEntry::tombstone(1000)) + .expect("should create tombstone"); + + let result = kv + .upsert_partner_id_if_exists(&ec_id, "ssp_x", "uid-1") + .expect("should not error on tombstone"); assert_eq!( - operations.inserts().len(), - 1, - "fresh create should only insert the live root" - ); - assert!( - kv.get(&ec_id) - .expect("should read entry") - .is_some_and(|(entry, _)| entry.consent.ok), - "fresh create should persist a live entry" + result, + UpsertResult::ConsentWithdrawn, + "should reject a partner upsert for a withdrawn identity" ); } #[test] - fn create_or_revive_clears_withdrawal_marker() { + fn snapshot_bulk_upsert_returns_persisted_entry_without_stale_generation() { let kv = KvIdentityGraph::in_memory("test_store"); let ec_id = format!("{}.ABC123", "a".repeat(64)); - kv.create(&ec_id, &live_entry()) - .expect("should create live entry"); + kv.create(&ec_id, &live_entry()).expect("should create"); let snapshot = kv.load_snapshot(&ec_id); - kv.tombstone_existing_from_snapshot(&ec_id, snapshot); - assert!( - kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should read withdrawal marker"), - "withdrawal should record completion" - ); + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - kv.create_or_revive(&ec_id, &live_entry()) - .expect("should revive tombstone"); + let outcome = kv.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - let (loaded, _) = kv - .get(&ec_id) - .expect("should read revived entry") - .expect("should find revived entry"); - assert!(loaded.consent.ok, "should be live after revive"); - assert!( - !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should read withdrawal marker"), - "revival should clear stale withdrawal completion" + let entry = outcome + .entry_for(&ec_id) + .expect("should retain persisted entry"); + assert_eq!( + entry.ids.get("ssp_x").map(|id| id.uid.as_str()), + Some("uid-1") + ); + assert_eq!( + outcome.generation_for(&ec_id), + None, + "backend does not return the post-write generation" ); } #[test] - fn stale_completion_marker_does_not_suppress_withdrawal_after_root_recreation() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::with_stale_lookups( - Arc::clone(&operations), - 1, - )); - let ec_id = snapshot_ec_id(); - let expired_updated = current_timestamp().saturating_sub(TOMBSTONE_TTL.as_secs() + 1); - let expired_tombstone = KvEntry::tombstone(expired_updated); - graph - .create(&ec_id, &expired_tombstone) - .expect("should seed old tombstone"); - graph - .write_withdrawal_marker(&ec_id, expired_updated) - .expect("should seed old completion marker"); - - // Model the root expiring just before its later-written completion - // marker, followed by recreation of the same key. - graph.store.delete(&ec_id).expect("should expire root row"); - assert_eq!( - graph - .create_if_absent(&ec_id, &live_entry()) - .expect("should recreate expired root"), - CreateIfAbsentOutcome::Written, - "should make the same key live again after root expiry" - ); + fn snapshot_bulk_upsert_does_not_create_missing_root() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - let outcome = graph.tombstone_existing_from_snapshot( + let outcome = kv.upsert_partner_ids_from_snapshot( &ec_id, + &updates, EcKvSnapshot::Missing { ec_id: ec_id.clone(), }, ); assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "an expired completion marker must not suppress withdrawal of a recreated live row" + matches!(outcome, EcKvSnapshot::Missing { .. }), + "missing snapshot should remain missing" ); - let (stored, _) = graph - .get(&ec_id) - .expect("should read recreated row") - .expect("should retain recreated row"); - assert!(!stored.consent.ok, "recreated row should be tombstoned"); + assert!(kv.get(&ec_id).expect("should read store").is_none()); } #[test] - fn withdrawal_failed_clock_stale_miss_tombstones_live_root() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::with_stale_lookups( - Arc::clone(&operations), - 1, - )); - let ec_id = snapshot_ec_id(); - graph - .create(&ec_id, &concurrent_live_entry()) - .expect("should seed a live row with partner IDs"); - graph - .write_withdrawal_marker(&ec_id, 1000) - .expect("should seed a completion marker left after root recreation"); - operations.reset(); - let missing = graph.load_snapshot(&ec_id); - assert!( - matches!(missing, EcKvSnapshot::Missing { .. }), - "should reproduce a stale point-read miss for the live row" - ); - - // None is the checked clock's failure result, not Unix epoch zero. - let outcome = graph.tombstone_unproven_missing(&ec_id, missing, None); + fn write_withdrawal_tombstone_overwrites_live_entry() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + kv.create(&ec_id, &live_entry()).expect("should create"); - let (stored, generation) = graph - .get(&ec_id) - .expect("should read back persisted withdrawal") - .expect("should retain the root"); - assert!( - !stored.consent.ok, - "an unusable clock must not let a marker suppress withdrawal" - ); - assert!( - stored.ids.is_empty(), - "withdrawal should clear stored partner IDs" - ); - assert_eq!( - generation, 2, - "withdrawal should write the existing root once" - ); - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "clock failure should not fail a strongly confirmed withdrawal" - ); - assert_eq!( - operations.exact_check_count(), - 1, - "should strongly check the root" - ); assert_eq!( - operations.inserts(), - vec![ - RecordedEcKvInsert { - mode: EcKvWriteMode::Overwrite, - ttl: TOMBSTONE_TTL - }, - RecordedEcKvInsert { - mode: EcKvWriteMode::Add, - ttl: TOMBSTONE_TTL - }, - ], - "should write only the root tombstone and its completion marker" + kv.write_withdrawal_tombstone(&ec_id, drop) + .expect("should write tombstone"), + TombstoneOutcome::Written, + "should tombstone an identity the store holds" ); + + let (loaded, _) = kv + .get(&ec_id) + .expect("should read entry back") + .expect("should find tombstone entry"); + assert!(!loaded.consent.ok, "should be withdrawn after tombstone"); } - #[test] - fn withdrawal_failed_clock_does_not_create_absent_root() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); - let ec_id = snapshot_ec_id(); - graph - .write_withdrawal_marker(&ec_id, 1000) - .expect("should seed a marker without a root"); - operations.reset(); - let missing = graph.load_snapshot(&ec_id); + // ----------------------------------------------------------------------- + // Snapshot-aware mutation stores and tests + // ----------------------------------------------------------------------- - let outcome = graph.tombstone_unproven_missing(&ec_id, missing, None); + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + struct RecordedEcKvInsert { + mode: EcKvWriteMode, + ttl: Duration, + } - assert!( - matches!(outcome, EcKvSnapshot::Missing { .. }), - "absent root should remain missing" - ); - assert!( - graph - .get(&ec_id) - .expect("should read absent root") - .is_none(), - "clock failure must not mint an unknown root" - ); - assert_eq!( - operations.exact_check_count(), - 1, - "should prove root absence despite the marker" - ); - assert!( - operations.inserts().is_empty(), - "absent identity should cause no writes" - ); + #[derive(Default)] + struct RecordedEcKvOperations { + lookups: std::sync::atomic::AtomicUsize, + exact_checks: std::sync::atomic::AtomicUsize, + inserts: std::sync::Mutex>, + lists: std::sync::atomic::AtomicUsize, + deletes: std::sync::atomic::AtomicUsize, } - #[test] - fn withdrawal_marker_validity_requires_a_usable_clock_before_expiry() { - let graph = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - graph - .write_withdrawal_marker(&ec_id, 1000) - .expect("should seed marker"); - let valid_until = 1000 + TOMBSTONE_TTL.as_secs(); + impl RecordedEcKvOperations { + fn reset(&self) { + self.lookups.store(0, std::sync::atomic::Ordering::Relaxed); + self.exact_checks + .store(0, std::sync::atomic::Ordering::Relaxed); + self.inserts + .lock() + .expect("should lock recorded inserts") + .clear(); + self.lists.store(0, std::sync::atomic::Ordering::Relaxed); + self.deletes.store(0, std::sync::atomic::Ordering::Relaxed); + } - for (now, expected) in [ - (Some(valid_until - 1), true), - (Some(valid_until), false), - (Some(valid_until + 1), false), - (None, false), - ] { - assert_eq!( - graph - .withdrawal_marker_exists(&ec_id, now) - .expect("should check marker validity"), - expected, - "marker validity should honor clock availability and exclusive expiry at {now:?}" - ); + fn lookup_count(&self) -> usize { + self.lookups.load(std::sync::atomic::Ordering::Relaxed) } - } - #[test] - fn withdrawal_marker_existence_rejects_malformed_expiry_suffix() { - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let marker_key = KvIdentityGraph::withdrawal_marker_key( - &ec_id, - current_timestamp() + TOMBSTONE_TTL.as_secs(), - ); - let longer_key = format!("{marker_key}-longer"); - let store = InMemoryEcKv::new("test_store"); - store - .insert( - &longer_key, - EcKvWrite { - body: "1", - metadata: "{}", - ttl: TOMBSTONE_TTL, - mode: EcKvWriteMode::Add, - }, - ) - .expect("should seed longer marker key"); - let kv = KvIdentityGraph::new(store); + fn operation_count(&self) -> usize { + self.lookup_count() + + self.exact_check_count() + + self.inserts().len() + + self.list_count() + + self.delete_count() + } - assert!( - !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should validate withdrawal marker expiry"), - "a malformed expiry suffix must not prove withdrawal completion" - ); - } + fn exact_check_count(&self) -> usize { + self.exact_checks.load(std::sync::atomic::Ordering::Relaxed) + } - #[test] - fn clear_absent_withdrawal_marker_lists_once_without_delete() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let kv = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); - let ec_id = snapshot_ec_id(); + fn list_count(&self) -> usize { + self.lists.load(std::sync::atomic::Ordering::Relaxed) + } - kv.clear_withdrawal_marker(&ec_id) - .expect("should accept an absent marker"); + fn delete_count(&self) -> usize { + self.deletes.load(std::sync::atomic::Ordering::Relaxed) + } - assert_eq!( - operations.list_count(), - 1, - "absent marker guard should list once" - ); - assert_eq!( - operations.delete_count(), - 0, - "absent marker guard should avoid a delete" - ); + fn inserts(&self) -> Vec { + self.inserts + .lock() + .expect("should lock recorded inserts") + .clone() + } } - #[test] - fn writing_an_existing_withdrawal_marker_is_idempotent() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let kv = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); - let ec_id = snapshot_ec_id(); - let updated = current_timestamp(); + /// In-memory store that records operations and can inject focused failures. + struct RecordingEcKv { + inner: InMemoryEcKv, + operations: Arc, + stale_lookups_remaining: std::sync::Mutex, + lag_live_entries: bool, + marker_operations_fail: bool, + root_check_fails: bool, + marker_delete_failure_removes_key: Option, + } - kv.write_withdrawal_marker(&ec_id, updated) - .expect("should write first completion marker"); - kv.write_withdrawal_marker(&ec_id, updated) - .expect("should accept completion marker add collision"); + impl RecordingEcKv { + fn new(operations: Arc) -> Self { + Self::with_stale_lookups(operations, 0) + } - assert_eq!( - operations.inserts(), - vec![ - RecordedEcKvInsert { - mode: EcKvWriteMode::Add, - ttl: TOMBSTONE_TTL, - }, - RecordedEcKvInsert { - mode: EcKvWriteMode::Add, - ttl: TOMBSTONE_TTL, - }, - ], - "both marker add attempts should reach the store" - ); - assert!( - kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should read completion marker"), - "completion marker should remain valid" - ); - } + fn with_stale_lookups(operations: Arc, stale_lookups: u32) -> Self { + Self { + inner: InMemoryEcKv::new("recording-store"), + operations, + stale_lookups_remaining: std::sync::Mutex::new(stale_lookups), + lag_live_entries: false, + marker_operations_fail: false, + root_check_fails: false, + marker_delete_failure_removes_key: None, + } + } - #[test] - fn marker_list_saturation_blocks_revival_and_hard_delete_before_mutation() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); - let ec_id = snapshot_ec_id(); - graph - .create(&ec_id, &KvEntry::tombstone(current_timestamp())) - .expect("should seed tombstone"); - let first_valid_until = current_timestamp() + TOMBSTONE_TTL.as_secs(); - for offset in 0..WITHDRAWAL_MARKER_LIST_LIMIT { - let marker_key = KvIdentityGraph::withdrawal_marker_key( - &ec_id, - first_valid_until + u64::from(offset), - ); - graph - .store + fn with_lagging_live_lookups(operations: Arc) -> Self { + Self { + lag_live_entries: true, + ..Self::new(operations) + } + } + + fn with_marker_failures( + operations: Arc, + stale_lookups: u32, + ) -> Self { + Self { + marker_operations_fail: true, + ..Self::with_stale_lookups(operations, stale_lookups) + } + } + + fn completed_with_root_check_failure( + operations: Arc, + ec_id: &str, + ) -> Self { + let store = Self { + root_check_fails: true, + stale_lookups_remaining: std::sync::Mutex::new(u32::MAX), + ..Self::new(operations) + }; + let tombstone = KvEntry::tombstone(current_timestamp()); + let (body, metadata) = + KvIdentityGraph::serialize_entry(&tombstone, store.inner.store_name()) + .expect("should serialize tombstone"); + store + .inner .insert( - &marker_key, + ec_id, + EcKvWrite { + body: &body, + metadata: &metadata, + ttl: TOMBSTONE_TTL, + mode: EcKvWriteMode::Add, + }, + ) + .expect("should seed tombstone"); + store + .inner + .insert( + &KvIdentityGraph::withdrawal_marker_key( + ec_id, + tombstone.consent.updated + TOMBSTONE_TTL.as_secs(), + ), EcKvWrite { body: "1", metadata: "{}", @@ -2946,250 +2567,306 @@ mod tests { }, ) .expect("should seed completion marker"); + store } - operations.reset(); - let result = graph.create_or_revive(&ec_id, &live_entry()); + fn with_marker_delete_failure( + operations: Arc, + remove_before_error: bool, + ) -> Self { + Self { + marker_delete_failure_removes_key: Some(remove_before_error), + ..Self::new(operations) + } + } - assert!( - result.is_err(), - "revival should fail when marker cleanup cannot prove the prefix is exhausted" - ); - let (stored, _) = graph - .get(&ec_id) - .expect("should read root") - .expect("should retain root"); - assert!(!stored.consent.ok, "root should remain tombstoned"); - assert_eq!( - operations.delete_count(), - 0, - "saturated marker cleanup should not make partial progress" - ); - - operations.reset(); - let delete_result = graph.delete(&ec_id); + fn marker_error(&self, operation: &str) -> Report { + Report::new(TrustedServerError::KvStore { + store_name: self.inner.store_name().to_owned(), + message: format!("completion marker {operation} failed"), + }) + } - assert!( - delete_result.is_err(), - "hard deletion should fail when marker cleanup cannot prove the prefix is exhausted" - ); - let (stored, _) = graph - .get(&ec_id) - .expect("should read root after rejected hard delete") - .expect("rejected hard delete should retain root"); - assert!( - !stored.consent.ok, - "rejected hard delete should leave the tombstoned root in place" - ); - assert_eq!( - operations.delete_count(), - 0, - "hard deletion should not remove the root before saturated marker cleanup fails" - ); + fn root_check_error(&self) -> Report { + Report::new(TrustedServerError::KvStore { + store_name: self.inner.store_name().to_owned(), + message: "root existence check failed".to_owned(), + }) + } } - #[test] - fn clear_withdrawal_marker_accepts_concurrent_removal_after_delete_error() { - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let operations = Arc::new(RecordedEcKvOperations::default()); - let kv = KvIdentityGraph::new(RecordingEcKv::with_marker_delete_failure(operations, true)); - kv.create(&ec_id, &live_entry()) - .expect("should create live entry"); - let snapshot = kv.load_snapshot(&ec_id); - kv.tombstone_existing_from_snapshot(&ec_id, snapshot); + impl EcKvStore for RecordingEcKv { + fn store_name(&self) -> &str { + self.inner.store_name() + } - kv.clear_withdrawal_marker(&ec_id) - .expect("should accept a marker removed by another request"); + fn lookup(&self, key: &str) -> Result, Report> { + self.operations + .lookups + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let mut stale_lookups = self + .stale_lookups_remaining + .lock() + .expect("should lock stale lookup counter"); + if *stale_lookups > 0 { + *stale_lookups -= 1; + return Ok(None); + } + let found = self.inner.lookup(key)?; + if self.lag_live_entries + && found.as_ref().is_some_and(|entry| { + serde_json::from_slice::(&entry.body) + .expect("should decode test entry") + .consent + .ok + }) + { + return Ok(None); + } + Ok(found) + } - assert!( - !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should confirm marker removal"), - "completion marker should remain absent" - ); + fn key_exists(&self, key: &str) -> Result> { + self.operations + .exact_checks + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if !key.starts_with(WITHDRAWAL_MARKER_PREFIX) && self.root_check_fails { + return Err(self.root_check_error()); + } + self.inner.key_exists(key) + } + + fn insert( + &self, + key: &str, + write: EcKvWrite<'_>, + ) -> Result> { + self.operations + .inserts + .lock() + .expect("should lock recorded inserts") + .push(RecordedEcKvInsert { + mode: write.mode, + ttl: write.ttl, + }); + if key.starts_with(WITHDRAWAL_MARKER_PREFIX) && self.marker_operations_fail { + return Err(self.marker_error("write")); + } + self.inner.insert(key, write) + } + + fn list_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result, Report> { + self.operations + .lists + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if prefix.starts_with(WITHDRAWAL_MARKER_PREFIX) && self.marker_operations_fail { + return Err(self.marker_error("lookup")); + } + self.inner.list_keys_with_prefix(prefix, limit) + } + + fn delete(&self, key: &str) -> Result<(), Report> { + self.operations + .deletes + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if key.starts_with(WITHDRAWAL_MARKER_PREFIX) + && let Some(remove_before_error) = self.marker_delete_failure_removes_key + { + if remove_before_error { + self.inner.delete(key)?; + } + return Err(self.marker_error("delete")); + } + self.inner.delete(key) + } } #[test] - fn clear_withdrawal_marker_preserves_delete_error_when_marker_remains() { - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let operations = Arc::new(RecordedEcKvOperations::default()); - let kv = KvIdentityGraph::new(RecordingEcKv::with_marker_delete_failure(operations, false)); - kv.create(&ec_id, &live_entry()) - .expect("should create live entry"); - let snapshot = kv.load_snapshot(&ec_id); - kv.tombstone_existing_from_snapshot(&ec_id, snapshot); + fn snapshot_upsert_with_generation_writes_without_reading() { + let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let graph = KvIdentityGraph::counting("counting-store", lookups.clone()); + let ec_id = snapshot_ec_id(); + graph.create(&ec_id, &live_entry()).expect("should seed"); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: Some(1), + }; + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - assert!( - kv.clear_withdrawal_marker(&ec_id).is_err(), - "a marker that remains after delete failure should block revival" + let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); + + assert_eq!( + lookups.load(std::sync::atomic::Ordering::Relaxed), + 0, + "a usable generation must avoid the initial read" ); - assert!( - kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should confirm marker remains"), - "completion marker should remain present" + assert_eq!( + outcome + .entry_for(&ec_id) + .and_then(|entry| entry.ids.get("ssp_x")) + .map(|id| id.uid.as_str()), + Some("uid-1") + ); + assert_eq!( + outcome.generation_for(&ec_id), + None, + "a successful write should clear the stale generation" ); } #[test] - fn withdrawal_marker_is_excluded_from_hash_prefix_count() { - let hash = "a".repeat(64); - let ec_id = format!("{hash}.ABC123"); + fn snapshot_upsert_unchanged_updates_preserve_generation() { let kv = KvIdentityGraph::in_memory("test_store"); - kv.create(&ec_id, &live_entry()) - .expect("should create live entry"); + let ec_id = snapshot_ec_id(); + let mut seeded = live_entry(); + apply_partner_id_updates(&mut seeded, &[PartnerIdUpdate::new("ssp_x", "uid-1")]); + kv.create(&ec_id, &seeded).expect("should seed"); + let snapshot = kv.load_snapshot(&ec_id); assert_eq!( - kv.count_hash_prefix_keys(&hash) - .expect("should count live root"), - 1, - "the root should be the only hash-prefix key" + snapshot.generation_for(&ec_id), + Some(1), + "seeded snapshot should carry its stored generation" ); - let snapshot = kv.load_snapshot(&ec_id); - kv.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!( - kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should read withdrawal marker"), - "withdrawal should record completion" + let outcome = kv.upsert_partner_ids_from_snapshot( + &ec_id, + &[PartnerIdUpdate::new("ssp_x", "uid-1")], + snapshot, ); + assert_eq!( - kv.count_hash_prefix_keys(&hash) - .expect("should count tombstoned root"), - 1, - "the marker namespace must not affect cluster counts" + outcome.generation_for(&ec_id), + Some(1), + "an unchanged merge preserves the usable generation and performs no write" ); } #[test] - fn delete_removes_withdrawal_marker() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - kv.create(&ec_id, &live_entry()) - .expect("should create live entry"); - let snapshot = kv.load_snapshot(&ec_id); - kv.tombstone_existing_from_snapshot(&ec_id, snapshot); + fn snapshot_upsert_refreshes_unavailable_generation_exactly_once() { + let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let graph = KvIdentityGraph::counting("counting-store", lookups.clone()); + let ec_id = snapshot_ec_id(); + graph.create(&ec_id, &live_entry()).expect("should seed"); + // Finalize-written style snapshot: entry known, generation unavailable. + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: None, + }; + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - kv.delete(&ec_id).expect("should delete entry and marker"); + let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - assert!( - kv.get(&ec_id).expect("should read store").is_none(), - "hard delete should remove the root" + assert_eq!( + lookups.load(std::sync::atomic::Ordering::Relaxed), + 1, + "an unavailable generation refreshes exactly once before CAS" ); assert!( - !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) - .expect("should read withdrawal marker"), - "hard delete should remove withdrawal completion" + outcome + .entry_for(&ec_id) + .is_some_and(|e| e.ids.contains_key("ssp_x")), + "refreshing an unavailable generation should persist the partner update" ); } #[test] - fn upsert_partner_id_if_exists_reports_missing_key() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - - let result = kv - .upsert_partner_id_if_exists(&ec_id, "ssp_x", "uid-1") - .expect("should not error on missing key"); - assert_eq!( - result, - UpsertResult::NotFound, - "should reject a partner upsert for an EC ID the store does not hold" - ); - } - - #[test] - fn upsert_partner_id_if_exists_writes_and_detects_unchanged() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - kv.create(&ec_id, &live_entry()).expect("should create"); + fn snapshot_upsert_gen_unavailable_survives_four_conflicts_then_writes() { + // A generation-unavailable snapshot (finalize-written style) refreshes + // once to obtain a usable generation. That refresh must not consume a + // CAS attempt, so all five write attempts remain: four conflicts + // followed by a successful fifth write still persist the update. + let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(4, false)); + let ec_id = snapshot_ec_id(); + graph.create(&ec_id, &live_entry()).expect("should seed"); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: None, + }; + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - let first = kv - .upsert_partner_id_if_exists(&ec_id, "ssp_x", "uid-1") - .expect("should write partner id"); - assert_eq!( - first, - UpsertResult::Written, - "should report the first partner upsert as written" - ); + let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - let second = kv - .upsert_partner_id_if_exists(&ec_id, "ssp_x", "uid-1") - .expect("should detect unchanged uid"); assert_eq!( - second, - UpsertResult::Unchanged, - "should report an identical partner upsert as unchanged" + outcome + .entry_for(&ec_id) + .and_then(|entry| entry.ids.get("ssp_x")) + .map(|id| id.uid.as_str()), + Some("uid-1"), + "the fifth CAS attempt must still succeed after a refresh and four conflicts" ); } #[test] - fn upsert_partner_id_if_exists_retries_cas_conflict() { - let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(1, false)); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - graph - .create(&ec_id, &live_entry()) - .expect("should seed live entry"); + fn snapshot_upsert_cas_conflict_remerges_concurrent_data() { + let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(1, true)); + let ec_id = snapshot_ec_id(); + graph.create(&ec_id, &live_entry()).expect("should seed"); + let snapshot = graph.load_snapshot(&ec_id); + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - let result = graph - .upsert_partner_id_if_exists(&ec_id, "ssp_x", "uid-1") - .expect("should retry and write after one generation conflict"); + let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - assert_eq!(result, UpsertResult::Written); - let (entry, _) = graph - .get(&ec_id) - .expect("should read persisted entry") - .expect("should retain entry after conflict"); + let entry = outcome + .entry_for(&ec_id) + .expect("should persist re-merged entry"); assert_eq!( entry.ids.get("ssp_x").map(|id| id.uid.as_str()), Some("uid-1"), - "should persist requested UID after retry" - ); - } - - #[test] - fn upsert_partner_id_if_exists_rejects_tombstone() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - kv.create(&ec_id, &KvEntry::tombstone(1000)) - .expect("should create tombstone"); - - let result = kv - .upsert_partner_id_if_exists(&ec_id, "ssp_x", "uid-1") - .expect("should not error on tombstone"); - assert_eq!( - result, - UpsertResult::ConsentWithdrawn, - "should reject a partner upsert for a withdrawn identity" + "conflict must re-merge our update onto the concurrently revived row" ); + assert!(entry.consent.ok, "concurrent revive keeps the row live"); } #[test] - fn snapshot_bulk_upsert_returns_persisted_entry_without_stale_generation() { + fn snapshot_upsert_revalidates_transient_missing_and_persists() { + // An eventually-consistent point read earlier in the request missed a + // row that exists. Named routes such as `/auction` never run orphan + // recovery, so this refresh is the request's only chance to persist the + // collected partner IDs. let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - kv.create(&ec_id, &live_entry()).expect("should create"); - let snapshot = kv.load_snapshot(&ec_id); + let ec_id = snapshot_ec_id(); + kv.create(&ec_id, &live_entry()).expect("should seed live"); let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - let outcome = kv.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); + let outcome = kv.upsert_partner_ids_from_snapshot( + &ec_id, + &updates, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, + ); - let entry = outcome - .entry_for(&ec_id) - .expect("should retain persisted entry"); assert_eq!( - entry.ids.get("ssp_x").map(|id| id.uid.as_str()), - Some("uid-1") + outcome + .entry_for(&ec_id) + .and_then(|entry| entry.ids.get("ssp_x").map(|id| id.uid.clone())), + Some("uid-1".to_owned()), + "a stale miss must be revalidated before the updates are dropped" ); + let (stored, _) = kv + .get(&ec_id) + .expect("should read store") + .expect("should keep the row"); assert_eq!( - outcome.generation_for(&ec_id), - None, - "backend does not return the post-write generation" + stored.ids.get("ssp_x").map(|id| id.uid.as_str()), + Some("uid-1"), + "the revalidated update must reach the store" ); } #[test] - fn snapshot_bulk_upsert_does_not_create_missing_root() { + fn snapshot_upsert_confirmed_missing_still_never_creates() { + // Revalidation only changes what a *stale* miss does. A row that is + // genuinely absent on the refresh must stay absent. let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); + let ec_id = snapshot_ec_id(); let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; let outcome = kv.upsert_partner_ids_from_snapshot( @@ -3202,951 +2879,747 @@ mod tests { assert!( matches!(outcome, EcKvSnapshot::Missing { .. }), - "missing snapshot should remain missing" + "a confirmed miss must stay missing" + ); + assert!( + kv.get(&ec_id).expect("should read store").is_none(), + "must not create a root entry for a missing key" ); - assert!(kv.get(&ec_id).expect("should read store").is_none()); } #[test] - fn tombstone_existing_from_snapshot_never_creates_missing_key() { + fn snapshot_upsert_failed_snapshot_is_not_revalidated() { + // A lookup that already errored is not retried on the hot path, even + // though the row exists. let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let snapshot = EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }; - - let outcome = kv.tombstone_existing_from_snapshot(&ec_id, snapshot); + let ec_id = snapshot_ec_id(); + kv.create(&ec_id, &live_entry()).expect("should seed live"); + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - assert!( - matches!(outcome, EcKvSnapshot::Missing { .. }), - "withdrawal should preserve the missing snapshot for an absent identity" + let outcome = kv.upsert_partner_ids_from_snapshot( + &ec_id, + &updates, + EcKvSnapshot::Failed { + ec_id: ec_id.clone(), + }, ); + assert!( - kv.get(&ec_id).expect("should read store").is_none(), - "withdrawal must not create a tombstone for an absent key" + matches!(outcome, EcKvSnapshot::Failed { .. }), + "a failed lookup must not be retried by partner enrichment" ); } #[test] - fn tombstone_existing_from_snapshot_uses_existing_generation() { + fn snapshot_upsert_rejects_tombstone() { let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - kv.create(&ec_id, &live_entry()).expect("should create"); + let ec_id = snapshot_ec_id(); + kv.create(&ec_id, &KvEntry::tombstone(1000)) + .expect("should seed tombstone"); let snapshot = kv.load_snapshot(&ec_id); + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - let outcome = kv.tombstone_existing_from_snapshot(&ec_id, snapshot); + let outcome = kv.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); assert!( outcome .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "should return the persisted tombstone" + .is_some_and(|entry| entry.ids.is_empty()), + "a tombstone must reject partner enrichment" ); let (stored, _) = kv .get(&ec_id) .expect("should read store") - .expect("should preserve existing key"); - assert!(!stored.consent.ok, "should persist withdrawal state"); + .expect("should keep the tombstone"); + assert!(stored.ids.is_empty(), "no update should reach the store"); } #[test] - fn write_withdrawal_tombstone_overwrites_live_entry() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - kv.create(&ec_id, &live_entry()).expect("should create"); + fn snapshot_upsert_store_failure_returns_failed_not_request_local() { + let graph = KvIdentityGraph::new(WriteFailingEcKv::new()); + let ec_id = snapshot_ec_id(); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: Some(1), + }; + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - assert_eq!( - kv.write_withdrawal_tombstone(&ec_id, drop) - .expect("should write tombstone"), - TombstoneOutcome::Written, - "should tombstone an identity the store holds" - ); + let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - let (loaded, _) = kv - .get(&ec_id) - .expect("should read entry back") - .expect("should find tombstone entry"); - assert!(!loaded.consent.ok, "should be withdrawn after tombstone"); + assert!( + matches!(outcome, EcKvSnapshot::Failed { .. }), + "a store write failure must not claim request-local IDs were persisted" + ); } - // ----------------------------------------------------------------------- - // Snapshot-aware mutation stores and tests - // ----------------------------------------------------------------------- + #[test] + fn upsert_partner_id_rejects_tombstone() { + let graph = KvIdentityGraph::in_memory("test-store"); + let ec_id = snapshot_ec_id(); + graph + .create(&ec_id, &KvEntry::tombstone(1_000)) + .expect("should seed tombstone"); - #[derive(Debug, Clone, Copy, PartialEq, Eq)] - struct RecordedEcKvInsert { - mode: EcKvWriteMode, - ttl: Duration, - } + let result = graph.upsert_partner_id(&ec_id, "ssp.example.com", "uid-1"); - #[derive(Default)] - struct RecordedEcKvOperations { - lookups: std::sync::atomic::AtomicUsize, - exact_checks: std::sync::atomic::AtomicUsize, - inserts: std::sync::Mutex>, - lists: std::sync::atomic::AtomicUsize, - deletes: std::sync::atomic::AtomicUsize, + assert!(result.is_err(), "public upsert should reject a tombstone"); + let (stored, _) = graph + .get(&ec_id) + .expect("should read store") + .expect("should preserve tombstone"); + assert!(!stored.consent.ok, "entry should remain withdrawn"); + assert!( + stored.ids.is_empty(), + "upsert should not repopulate partner IDs" + ); } - impl RecordedEcKvOperations { - fn reset(&self) { - self.lookups.store(0, std::sync::atomic::Ordering::Relaxed); - self.exact_checks - .store(0, std::sync::atomic::Ordering::Relaxed); - self.inserts - .lock() - .expect("should lock recorded inserts") - .clear(); - self.lists.store(0, std::sync::atomic::Ordering::Relaxed); - self.deletes.store(0, std::sync::atomic::Ordering::Relaxed); - } - - fn lookup_count(&self) -> usize { - self.lookups.load(std::sync::atomic::Ordering::Relaxed) - } + #[test] + fn key_exists_confirmed_distinguishes_absence_from_a_stale_point_read() { + let graph = KvIdentityGraph::stale_lookup("stale-store", 1); + let ec_id = snapshot_ec_id(); + graph + .create(&ec_id, &live_entry()) + .expect("should seed live"); - fn exact_check_count(&self) -> usize { - self.exact_checks.load(std::sync::atomic::Ordering::Relaxed) - } + assert!( + graph.get(&ec_id).expect("should read store").is_none(), + "the first point read is stale by construction" + ); + assert!( + graph + .key_exists_confirmed(&ec_id) + .expect("should list the store"), + "the list API must still see a row the point read missed" + ); - fn list_count(&self) -> usize { - self.lists.load(std::sync::atomic::Ordering::Relaxed) - } + let absent = KvIdentityGraph::in_memory("empty-store"); + assert!( + !absent + .key_exists_confirmed(&ec_id) + .expect("should list the store"), + "an empty store must prove absence" + ); + } - fn delete_count(&self) -> usize { - self.deletes.load(std::sync::atomic::Ordering::Relaxed) - } + #[test] + fn snapshot_upsert_keeps_add_confirmed_present_when_refresh_misses() { + // The page-load write records its successful `Add` create as `Present` + // without a generation. EID ingestion refreshes that snapshot to obtain one; on an + // eventually-consistent store the refresh can miss. Enrichment is best + // effort, so the miss must not retract the confirmed create — otherwise + // finalization suppresses the `ts-ec` cookie for a root that was written. + let graph = KvIdentityGraph::in_memory("empty-store"); + let ec_id = snapshot_ec_id(); + let add_confirmed = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: None, + }; + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - fn operation_count(&self) -> usize { - self.lookup_count() - + self.exact_check_count() - + self.inserts().len() - + self.list_count() - + self.delete_count() - } + let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, add_confirmed); - fn inserts(&self) -> Vec { - self.inserts - .lock() - .expect("should lock recorded inserts") - .clone() - } + assert!( + outcome.entry_for(&ec_id).is_some(), + "an Add-confirmed row must survive a non-authoritative refresh miss" + ); + assert!( + graph.get(&ec_id).expect("should read store").is_none(), + "a missed refresh must not create or overwrite a root" + ); } - /// In-memory store that records operations and can inject focused failures. - struct RecordingEcKv { - inner: InMemoryEcKv, - operations: Arc, - stale_lookups_remaining: std::sync::Mutex, - lag_live_entries: bool, - marker_operations_fail: bool, - root_check_fails: bool, - marker_delete_failure_removes_key: Option, - } + #[test] + fn snapshot_upsert_keeps_add_confirmed_present_when_refresh_fails() { + let graph = KvIdentityGraph::failing("failing-store"); + let ec_id = snapshot_ec_id(); + let add_confirmed = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: None, + }; + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - impl RecordingEcKv { - fn new(operations: Arc) -> Self { - Self::with_stale_lookups(operations, 0) - } + let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, add_confirmed); - fn with_stale_lookups(operations: Arc, stale_lookups: u32) -> Self { - Self { - inner: InMemoryEcKv::new("recording-store"), - operations, - stale_lookups_remaining: std::sync::Mutex::new(stale_lookups), - lag_live_entries: false, - marker_operations_fail: false, - root_check_fails: false, - marker_delete_failure_removes_key: None, - } - } + assert!( + outcome.entry_for(&ec_id).is_some(), + "a read failure is not evidence of absence and must not retract the create" + ); + } - fn with_lagging_live_lookups(operations: Arc) -> Self { - Self { - lag_live_entries: true, - ..Self::new(operations) - } - } + #[test] + fn snapshot_upsert_without_proof_still_reports_a_refresh_miss() { + // No prior proof of existence: a `NotRead` snapshot that refreshes into + // a miss must stay `Missing` so finalization can run orphan recovery. + let graph = KvIdentityGraph::in_memory("empty-store"); + let ec_id = snapshot_ec_id(); + let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - fn with_marker_failures( - operations: Arc, - stale_lookups: u32, - ) -> Self { - Self { - marker_operations_fail: true, - ..Self::with_stale_lookups(operations, stale_lookups) - } - } + let outcome = + graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, EcKvSnapshot::NotRead); - fn completed_with_root_check_failure( - operations: Arc, - ec_id: &str, - ) -> Self { - let store = Self { - root_check_fails: true, - stale_lookups_remaining: std::sync::Mutex::new(u32::MAX), - ..Self::new(operations) - }; - let tombstone = KvEntry::tombstone(current_timestamp()); - let (body, metadata) = - KvIdentityGraph::serialize_entry(&tombstone, store.inner.store_name()) - .expect("should serialize tombstone"); - store - .inner - .insert( - ec_id, - EcKvWrite { - body: &body, - metadata: &metadata, - ttl: TOMBSTONE_TTL, - mode: EcKvWriteMode::Add, - }, - ) - .expect("should seed tombstone"); - store - .inner - .insert( - &KvIdentityGraph::withdrawal_marker_key( - ec_id, - tombstone.consent.updated + TOMBSTONE_TTL.as_secs(), - ), - EcKvWrite { - body: "1", - metadata: "{}", - ttl: TOMBSTONE_TTL, - mode: EcKvWriteMode::Add, - }, - ) - .expect("should seed completion marker"); - store - } + assert!( + matches!(outcome, EcKvSnapshot::Missing { .. }), + "an unproven refresh miss must remain a miss" + ); + } - fn with_marker_delete_failure( - operations: Arc, - remove_before_error: bool, - ) -> Self { - Self { - marker_delete_failure_removes_key: Some(remove_before_error), - ..Self::new(operations) - } - } + #[test] + fn a_store_error_never_carries_the_whole_identifier() { + // Every message in this module goes through `log_id`, so a report that + // reaches a log cannot disclose the identifier it is about. + let kv = KvIdentityGraph::failing("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); - fn marker_error(&self, operation: &str) -> Report { - Report::new(TrustedServerError::KvStore { - store_name: self.inner.store_name().to_owned(), - message: format!("completion marker {operation} failed"), - }) - } + let report = kv + .create(&ec_id, &live_entry()) + .expect_err("the failing store should error"); - fn root_check_error(&self) -> Report { - Report::new(TrustedServerError::KvStore { - store_name: self.inner.store_name().to_owned(), - message: "root existence check failed".to_owned(), - }) - } + let rendered = format!("{report:?}"); + assert!( + !rendered.contains(&ec_id), + "a store error must not disclose the identifier: {rendered}" + ); } - impl EcKvStore for RecordingEcKv { - fn store_name(&self) -> &str { - self.inner.store_name() - } - - fn lookup(&self, key: &str) -> Result, Report> { - self.operations - .lookups - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let mut stale_lookups = self - .stale_lookups_remaining - .lock() - .expect("should lock stale lookup counter"); - if *stale_lookups > 0 { - *stale_lookups -= 1; - return Ok(None); - } - let found = self.inner.lookup(key)?; - if self.lag_live_entries - && found.as_ref().is_some_and(|entry| { - serde_json::from_slice::(&entry.body) - .expect("should decode test entry") - .consent - .ok - }) - { - return Ok(None); - } - Ok(found) - } - - fn key_exists(&self, key: &str) -> Result> { - self.operations - .exact_checks - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - if !key.starts_with(WITHDRAWAL_MARKER_PREFIX) && self.root_check_fails { - return Err(self.root_check_error()); - } - self.inner.key_exists(key) - } + #[test] + fn a_locally_built_error_never_carries_the_whole_identifier() { + // The injected-failure case above covers errors the backend produces. + // These are built in this module from the identifier itself, on every + // path a request can reach: a duplicate create, single upserts naming a + // key the store does not hold or has withdrawn, and the CAS-exhaustion + // terminal errors. + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + kv.create(&ec_id, &live_entry()).expect("should create"); - fn insert( - &self, - key: &str, - write: EcKvWrite<'_>, - ) -> Result> { - self.operations - .inserts - .lock() - .expect("should lock recorded inserts") - .push(RecordedEcKvInsert { - mode: write.mode, - ttl: write.ttl, - }); - if key.starts_with(WITHDRAWAL_MARKER_PREFIX) && self.marker_operations_fail { - return Err(self.marker_error("write")); - } - self.inner.insert(key, write) - } + let duplicate = kv + .create(&ec_id, &live_entry()) + .expect_err("a second create should be refused"); + let missing = kv + .upsert_partner_id(&format!("{}.ZZZ999", "b".repeat(64)), "partner", "uid") + .expect_err("an upsert on a missing key should be refused"); + let withdrawn = { + assert_eq!( + kv.write_withdrawal_tombstone(&ec_id, drop) + .expect("should tombstone"), + TombstoneOutcome::Written, + "should tombstone the seeded identity" + ); + kv.upsert_partner_id(&ec_id, "partner", "uid") + .expect_err("an upsert on a withdrawn key should be refused") + }; - fn list_keys_with_prefix( - &self, - prefix: &str, - limit: u32, - ) -> Result, Report> { - self.operations - .lists - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - if prefix.starts_with(WITHDRAWAL_MARKER_PREFIX) && self.marker_operations_fail { - return Err(self.marker_error("lookup")); - } - self.inner.list_keys_with_prefix(prefix, limit) - } + // The remaining CAS-exhaustion paths build their message the same way, + // and a store that never lets a write land is the only way to reach them. + let cas_revive = { + let store = ConflictInjectingEcKv::new(MAX_CAS_RETRIES + 1, false); + store.seed_tombstone(&ec_id); + KvIdentityGraph::new(store) + .create_or_revive(&ec_id, &live_entry()) + .expect_err("should exhaust CAS retries") + }; + let cas_upsert = { + let store = ConflictInjectingEcKv::new(MAX_CAS_RETRIES + 1, false); + store.seed_live(&ec_id); + KvIdentityGraph::new(store) + .upsert_partner_id(&ec_id, "partner", "uid") + .expect_err("should exhaust CAS retries") + }; + let cas_if_exists = { + let store = ConflictInjectingEcKv::new(MAX_CAS_RETRIES + 1, false); + store.seed_live(&ec_id); + KvIdentityGraph::new(store) + .upsert_partner_id_if_exists(&ec_id, "partner", "uid") + .expect_err("should exhaust CAS retries") + }; - fn delete(&self, key: &str) -> Result<(), Report> { - self.operations - .deletes - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - if key.starts_with(WITHDRAWAL_MARKER_PREFIX) - && let Some(remove_before_error) = self.marker_delete_failure_removes_key - { - if remove_before_error { - self.inner.delete(key)?; - } - return Err(self.marker_error("delete")); - } - self.inner.delete(key) + for (label, report) in [ + ("duplicate create", duplicate), + ("missing key", missing), + ("withdrawn key", withdrawn), + ("CAS exhaustion reviving", cas_revive), + ("CAS exhaustion upserting", cas_upsert), + ("CAS exhaustion upserting if present", cas_if_exists), + ] { + let rendered = format!("{report:?}"); + assert!( + !rendered.contains(&ec_id) && !rendered.contains(&"b".repeat(64)), + "the {label} error must not disclose the identifier: {rendered}" + ); } } - /// [`EcKvStore`] whose reads succeed but every write fails, simulating a - /// store that becomes unwritable mid-request. - struct WriteFailingEcKv { - inner: InMemoryEcKv, - } + #[test] + fn write_withdrawal_tombstone_ignores_an_identity_the_store_does_not_hold() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "b".repeat(64)); - impl WriteFailingEcKv { - fn new() -> Self { - Self { - inner: InMemoryEcKv::new("write-failing-store"), - } - } + assert_eq!( + kv.write_withdrawal_tombstone(&ec_id, drop) + .expect("should resolve the withdrawal"), + TombstoneOutcome::UnknownIdentity, + "an identity that was never issued has nothing to withdraw" + ); + assert!( + kv.get(&ec_id).expect("should read back").is_none(), + "should not create a row for an identity the store never held" + ); } - impl EcKvStore for WriteFailingEcKv { - fn store_name(&self) -> &str { - self.inner.store_name() - } - fn lookup(&self, key: &str) -> Result, Report> { - self.inner.lookup(key) - } - fn key_exists(&self, key: &str) -> Result> { - self.inner.key_exists(key) - } + #[test] + fn withdrawing_many_unheld_identities_creates_no_rows() { + let kv = KvIdentityGraph::in_memory("test_store"); + let hash = "c".repeat(64); - fn insert( - &self, - _key: &str, - _write: EcKvWrite<'_>, - ) -> Result> { - Err(Report::new(TrustedServerError::KvStore { - store_name: self.inner.store_name().to_owned(), - message: "write failing test store".to_owned(), - })) - } - fn list_keys_with_prefix( - &self, - prefix: &str, - limit: u32, - ) -> Result, Report> { - self.inner.list_keys_with_prefix(prefix, limit) - } - fn delete(&self, key: &str) -> Result<(), Report> { - self.inner.delete(key) + // The suffix is caller-supplied, so a shared hash prefix must not be + // enough to have a row written under it. + for suffix in ["aaaaaa", "bbbbbb", "cccccc", "dddddd"] { + assert_eq!( + kv.write_withdrawal_tombstone(&format!("{hash}.{suffix}"), drop) + .expect("should resolve the withdrawal"), + TombstoneOutcome::UnknownIdentity, + "suffix `{suffix}` was never issued" + ); } + + assert_eq!( + kv.count_hash_prefix_keys(&hash) + .expect("should count the prefix"), + 0, + "should hold no rows under a hash nothing was issued for" + ); } #[test] - fn snapshot_upsert_with_generation_writes_without_reading() { - let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let graph = KvIdentityGraph::counting("counting-store", lookups.clone()); - let ec_id = snapshot_ec_id(); - graph.create(&ec_id, &live_entry()).expect("should seed"); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: Some(1), - }; - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - + fn withdrawal_does_not_lose_a_new_identity_to_lookup_lag() { + let operations = Arc::new(RecordedEcKvOperations::default()); + let kv = KvIdentityGraph::new(RecordingEcKv::with_lagging_live_lookups(operations)); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + kv.create(&ec_id, &live_entry()) + .expect("should issue an identity"); + assert!( + kv.lookup_raw(&ec_id) + .expect("should read lagging replica") + .is_none(), + "should model the issuance replication gap" + ); assert_eq!( - lookups.load(std::sync::atomic::Ordering::Relaxed), - 0, - "a usable generation must avoid the initial read" + kv.write_withdrawal_tombstone(&ec_id, drop) + .expect("should process withdrawal"), + TombstoneOutcome::Written, + "should tombstone an issued identity despite a lagging lookup" ); assert_eq!( - outcome - .entry_for(&ec_id) - .and_then(|entry| entry.ids.get("ssp_x")) - .map(|id| id.uid.as_str()), - Some("uid-1") + kv.upsert_partner_id_if_exists(&ec_id, "partner", "uid") + .expect("should check batch-sync eligibility"), + UpsertResult::ConsentWithdrawn, + "should not revive the withdrawn identity through batch sync" + ); + } + + #[test] + fn a_withdrawal_checks_strong_existence_once_without_eventual_lookup() { + // One existence operation may page at the backend, but must not be + // followed by an eventually consistent lookup. + let operations = Arc::new(RecordedEcKvOperations::default()); + let kv = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); + + let absent = format!("{}.ABC123", "e".repeat(64)); + assert_eq!( + kv.write_withdrawal_tombstone(&absent, drop) + .expect("should resolve the withdrawal"), + TombstoneOutcome::UnknownIdentity, + "an absent identity is not held" ); assert_eq!( - outcome.generation_for(&ec_id), - None, - "a successful write should clear the stale generation" + operations.exact_check_count(), + 1, + "an unknown identity should cost one strong read" ); - } - #[test] - fn snapshot_upsert_unchanged_updates_preserve_generation() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - let mut seeded = live_entry(); - apply_partner_id_updates(&mut seeded, &[PartnerIdUpdate::new("ssp_x", "uid-1")]); - kv.create(&ec_id, &seeded).expect("should seed"); - let snapshot = kv.load_snapshot(&ec_id); + let held = format!("{}.ABC123", "a".repeat(64)); + kv.create(&held, &live_entry()).expect("should create"); + let before = operations.exact_check_count(); assert_eq!( - snapshot.generation_for(&ec_id), - Some(1), - "seeded snapshot should carry its stored generation" + kv.write_withdrawal_tombstone(&held, drop) + .expect("should resolve the withdrawal"), + TombstoneOutcome::Written, + "a held identity is tombstoned" ); - - let outcome = kv.upsert_partner_ids_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "uid-1")], - snapshot, + assert_eq!( + operations.exact_check_count() - before, + 1, + "a held identity is checked once, then written" ); - assert_eq!( - outcome.generation_for(&ec_id), - Some(1), - "an unchanged merge preserves the usable generation and performs no write" + operations.lookup_count(), + 0, + "should not use an eventual lookup for withdrawal" ); } #[test] - fn snapshot_upsert_refreshes_unavailable_generation_exactly_once() { - let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let graph = KvIdentityGraph::counting("counting-store", lookups.clone()); - let ec_id = snapshot_ec_id(); - graph.create(&ec_id, &live_entry()).expect("should seed"); - // Finalize-written style snapshot: entry known, generation unavailable. - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: None, - }; - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); + fn write_withdrawal_tombstone_refuses_an_empty_identifier() { + let kv = KvIdentityGraph::in_memory("test_store"); + kv.create(&format!("{}.ABC123", "f".repeat(64)), &live_entry()) + .expect("should create"); assert_eq!( - lookups.load(std::sync::atomic::Ordering::Relaxed), - 1, - "an unavailable generation refreshes exactly once before CAS" + kv.write_withdrawal_tombstone("", drop) + .expect("should resolve the withdrawal"), + TombstoneOutcome::UnknownIdentity, + "an empty identifier names no key and must not withdraw anything" ); + let (held, _) = kv + .get(&format!("{}.ABC123", "f".repeat(64))) + .expect("should read back") + .expect("should still hold the identity"); assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|e| e.ids.contains_key("ssp_x")), - "refreshing an unavailable generation should persist the partner update" + held.consent.ok, + "should not have withdrawn an unrelated row" ); } - #[test] - fn snapshot_upsert_gen_unavailable_survives_four_conflicts_then_writes() { - // A generation-unavailable snapshot (finalize-written style) refreshes - // once to obtain a usable generation. That refresh must not consume a - // CAS attempt, so all five write attempts remain: four conflicts - // followed by a successful fifth write still persist the update. - let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(4, false)); - let ec_id = snapshot_ec_id(); - graph.create(&ec_id, &live_entry()).expect("should seed"); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: None, - }; - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); + /// Store double whose reads always fail while writes still work. + struct ReadFailingEcKv { + inner: super::super::kv_backend::test_support::InMemoryEcKv, + } - assert_eq!( - outcome - .entry_for(&ec_id) - .and_then(|entry| entry.ids.get("ssp_x")) - .map(|id| id.uid.as_str()), - Some("uid-1"), - "the fifth CAS attempt must still succeed after a refresh and four conflicts" - ); + impl ReadFailingEcKv { + fn new() -> Self { + Self { + inner: super::super::kv_backend::test_support::InMemoryEcKv::new("test_store"), + } + } } - #[test] - fn snapshot_upsert_cas_conflict_remerges_concurrent_data() { - let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(1, true)); - let ec_id = snapshot_ec_id(); - graph.create(&ec_id, &live_entry()).expect("should seed"); - let snapshot = graph.load_snapshot(&ec_id); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; + impl EcKvStore for ReadFailingEcKv { + fn store_name(&self) -> &str { + self.inner.store_name() + } - let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); + fn lookup(&self, _key: &str) -> Result, Report> { + Err(Report::new(TrustedServerError::KvStore { + store_name: "test_store".to_owned(), + message: "reads unavailable".to_owned(), + })) + } - let entry = outcome - .entry_for(&ec_id) - .expect("should persist re-merged entry"); - assert_eq!( - entry.ids.get("ssp_x").map(|id| id.uid.as_str()), - Some("uid-1"), - "conflict must re-merge our update onto the concurrently revived row" - ); - assert!(entry.consent.ok, "concurrent revive keeps the row live"); + fn key_exists(&self, key: &str) -> Result> { + self.lookup(key).map(|entry| entry.is_some()) + } + + fn insert( + &self, + key: &str, + write: EcKvWrite<'_>, + ) -> Result> { + self.inner.insert(key, write) + } + + // Left working so a test can prove no row was created without going + // through the read path it just made fail. + fn list_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result, Report> { + self.inner.list_keys_with_prefix(prefix, limit) + } + + fn delete(&self, key: &str) -> Result<(), Report> { + self.inner.delete(key) + } } #[test] - fn snapshot_upsert_revalidates_transient_missing_and_persists() { - // An eventually-consistent point read earlier in the request missed a - // row that exists. Named routes such as `/auction` never run orphan - // recovery, so this refresh is the request's only chance to persist the - // collected partner IDs. - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - kv.create(&ec_id, &live_entry()).expect("should seed live"); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = kv.upsert_partner_ids_from_snapshot( - &ec_id, - &updates, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); + fn a_failing_check_is_not_a_way_to_write_for_an_identity_that_was_never_issued() { + // The caller controls the identifier and can drive load, so a store + // failure must not become a route to the write this gate exists to + // prevent. + let kv = KvIdentityGraph::new(ReadFailingEcKv::new()); + let hash = "8".repeat(64); + let ec_id = format!("{hash}.ABC123"); - assert_eq!( - outcome - .entry_for(&ec_id) - .and_then(|entry| entry.ids.get("ssp_x").map(|id| id.uid.clone())), - Some("uid-1".to_owned()), - "a stale miss must be revalidated before the updates are dropped" + assert!( + kv.write_withdrawal_tombstone(&ec_id, drop).is_err(), + "a check that cannot answer is a fault, so a caller inspecting only \ + the error case still reports it" ); - let (stored, _) = kv - .get(&ec_id) - .expect("should read store") - .expect("row should remain"); assert_eq!( - stored.ids.get("ssp_x").map(|id| id.uid.as_str()), - Some("uid-1"), - "the revalidated update must reach the store" + kv.count_hash_prefix_keys(&hash) + .expect("should count the prefix"), + 0, + "should not create a row while the store is degraded" ); } #[test] - fn snapshot_upsert_confirmed_missing_still_never_creates() { - // Revalidation only changes what a *stale* miss does. A row that is - // genuinely absent on the refresh must stay absent. - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; + fn a_caller_that_only_inspects_the_error_case_still_sees_a_failed_check() { + // The withdrawal call site is edited by more than one branch. Reporting + // a failed check through `Err` means the common + // `if let Err(..) = ...` shape cannot discard it, where a third `Ok` + // variant would be dropped without a compiler complaint. + let kv = KvIdentityGraph::new(ReadFailingEcKv::new()); + let ec_id = format!("{}.ABC123", "6".repeat(64)); - let outcome = kv.upsert_partner_ids_from_snapshot( - &ec_id, - &updates, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); + let mut reported = false; + if let Err(_err) = kv.write_withdrawal_tombstone(&ec_id, drop) { + reported = true; + } - assert!( - matches!(outcome, EcKvSnapshot::Missing { .. }), - "a confirmed miss must stay missing" - ); - assert!( - kv.get(&ec_id).expect("should read store").is_none(), - "must not create a root entry for a missing key" - ); + assert!(reported, "a failed check must reach an error-only caller"); } #[test] - fn snapshot_upsert_failed_snapshot_is_not_revalidated() { - // A lookup that already errored is not retried on the hot path, even - // though the row exists. + fn a_longer_key_does_not_answer_for_the_identity_it_starts_with() { let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - kv.create(&ec_id, &live_entry()).expect("should seed live"); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = kv.upsert_partner_ids_from_snapshot( - &ec_id, - &updates, - EcKvSnapshot::Failed { - ec_id: ec_id.clone(), - }, - ); + let ec_id = format!("{}.ABC123", "7".repeat(64)); + // Only a longer key exists. The identity itself was never issued, so a + // check that matched by prefix would report it as held and tombstone it. + kv.create(&format!("{ec_id}trailing"), &live_entry()) + .expect("should create"); assert!( - matches!(outcome, EcKvSnapshot::Failed { .. }), - "a failed lookup must not be retried by partner enrichment" + !kv.key_exists_confirmed(&ec_id).expect("should check"), + "a longer key is a different identity" + ); + assert_eq!( + kv.write_withdrawal_tombstone(&ec_id, drop) + .expect("should resolve the withdrawal"), + TombstoneOutcome::UnknownIdentity, + "should not tombstone an identity the store never held" + ); + assert!( + kv.get(&ec_id).expect("should read back").is_none(), + "should not create a row via a prefix match" ); } #[test] - fn snapshot_upsert_rejects_tombstone() { + fn key_exists_confirmed_distinguishes_held_identities() { let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - kv.create(&ec_id, &KvEntry::tombstone(1000)) - .expect("should seed tombstone"); - let snapshot = kv.load_snapshot(&ec_id); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = kv.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); + let held = format!("{}.ABC123", "d".repeat(64)); + let sibling = format!("{}.ZZZ999", "d".repeat(64)); + kv.create(&held, &live_entry()).expect("should create"); assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| entry.ids.is_empty()), - "a tombstone must reject partner enrichment" + kv.key_exists_confirmed(&held).expect("should check"), + "should confirm a held identity" + ); + assert!( + !kv.key_exists_confirmed(&sibling).expect("should check"), + "a different suffix under the same hash is a different identity" ); - let (stored, _) = kv - .get(&ec_id) - .expect("should read store") - .expect("tombstone should remain"); - assert!(stored.ids.is_empty(), "no update should reach the store"); } - #[test] - fn snapshot_upsert_store_failure_returns_failed_not_request_local() { - let graph = KvIdentityGraph::new(WriteFailingEcKv::new()); - let ec_id = snapshot_ec_id(); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: Some(1), - }; - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - + fn tombstone_via_page_load(kv: &KvIdentityGraph, ec_id: &str) { + let outcome = kv.tombstone_existing_from_snapshot(ec_id, EcKvSnapshot::NotRead); assert!( - matches!(outcome, EcKvSnapshot::Failed { .. }), - "a store write failure must not claim request-local IDs were persisted" + outcome + .entry_for(ec_id) + .is_some_and(|entry| !entry.consent.ok), + "should write tombstone" ); } #[test] - fn tombstone_existing_from_snapshot_skips_backend_for_authoritative_tombstone() { - for generation in [Some(7), None] { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); - let ec_id = snapshot_ec_id(); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(KvEntry::tombstone(1_000)), - generation, - }; + fn create_or_revive_clears_withdrawal_marker() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + kv.create(&ec_id, &live_entry()) + .expect("should create live entry"); + tombstone_via_page_load(&kv, &ec_id); + assert!( + kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) + .expect("should read withdrawal marker"), + "withdrawal should record completion" + ); - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot.clone()); + kv.create_or_revive(&ec_id, &live_entry()) + .expect("should revive tombstone"); - assert_eq!( - outcome, snapshot, - "should preserve authoritative tombstone state" - ); - assert_eq!( - operations.operation_count(), - 0, - "an authoritative tombstone should not access the backend" - ); - } + let (loaded, _) = kv + .get(&ec_id) + .expect("should read revived entry") + .expect("should find revived entry"); + assert!(loaded.consent.ok, "should be live after revive"); + assert!( + !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) + .expect("should read withdrawal marker"), + "revival should clear stale withdrawal completion" + ); } #[test] - fn tombstone_existing_from_snapshot_repeated_request_preserves_first_write() { + fn clear_withdrawal_marker_accepts_concurrent_removal_after_delete_error() { + let ec_id = format!("{}.ABC123", "a".repeat(64)); let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); - let ec_id = snapshot_ec_id(); - graph - .create(&ec_id, &live_entry()) - .expect("should seed live row"); - let live_snapshot = graph.load_snapshot(&ec_id); - operations.reset(); - - graph.tombstone_existing_from_snapshot(&ec_id, live_snapshot); - - assert_eq!( - operations.lookup_count(), - 0, - "usable generation should avoid a read" - ); - assert_eq!( - operations.inserts(), - vec![ - RecordedEcKvInsert { - mode: EcKvWriteMode::IfGenerationMatch(1), - ttl: TOMBSTONE_TTL, - }, - RecordedEcKvInsert { - mode: EcKvWriteMode::Add, - ttl: TOMBSTONE_TTL, - }, - ], - "first withdrawal should write the root and its completion marker" - ); - let first_snapshot = graph.load_snapshot(&ec_id); - let (first_entry, first_generation) = match &first_snapshot { - EcKvSnapshot::Present { - entry, generation, .. - } => (entry.as_ref().clone(), *generation), - other => panic!("should load first tombstone, got {other:?}"), - }; - operations.reset(); + let kv = KvIdentityGraph::new(RecordingEcKv::with_marker_delete_failure(operations, true)); + kv.create(&ec_id, &live_entry()) + .expect("should create live entry"); + tombstone_via_page_load(&kv, &ec_id); - let second_outcome = graph.tombstone_existing_from_snapshot(&ec_id, first_snapshot); + kv.clear_withdrawal_marker(&ec_id) + .expect("should accept a marker removed by another request"); - assert_eq!( - operations.operation_count(), - 0, - "repeated withdrawal should not access the backend" - ); - assert_eq!( - second_outcome.generation_for(&ec_id), - first_generation, - "repeated withdrawal should preserve the stored generation" - ); - assert_eq!( - second_outcome - .entry_for(&ec_id) - .map(|entry| entry.consent.updated), - Some(first_entry.consent.updated), - "repeated withdrawal should preserve the first tombstone timestamp" + assert!( + !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) + .expect("should confirm marker removal"), + "completion marker should remain absent" ); } #[test] - fn tombstone_existing_from_repeated_stale_miss_preserves_first_write() { + fn clear_withdrawal_marker_preserves_delete_error_when_marker_remains() { + let ec_id = format!("{}.ABC123", "a".repeat(64)); let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::with_stale_lookups( - Arc::clone(&operations), - 2, - )); - let ec_id = snapshot_ec_id(); - graph - .create(&ec_id, &live_entry()) - .expect("should seed live row"); - operations.reset(); - - let first_outcome = graph.tombstone_existing_from_snapshot( - &ec_id, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); - let first_updated = first_outcome - .entry_for(&ec_id) - .expect("should return first tombstone") - .consent - .updated; - assert_eq!( - operations.lookup_count(), - 1, - "first stale miss should retry its point read once" - ); - assert_eq!( - operations.list_count(), - 1, - "first stale miss should list completion markers once" - ); - assert_eq!( - operations.exact_check_count(), - 1, - "first stale miss should confirm root existence once" - ); - assert_eq!( - operations.inserts().len(), - 2, - "first stale miss should write the root and completion marker" - ); - operations.reset(); - - graph.tombstone_existing_from_snapshot( - &ec_id, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); + let kv = KvIdentityGraph::new(RecordingEcKv::with_marker_delete_failure(operations, false)); + kv.create(&ec_id, &live_entry()) + .expect("should create live entry"); + tombstone_via_page_load(&kv, &ec_id); - assert_eq!( - operations.lookup_count(), - 1, - "a repeated stale miss should retry its point read once" - ); - assert_eq!( - operations.exact_check_count(), - 0, - "a valid completion marker should avoid a root existence check" - ); - assert_eq!( - operations.list_count(), - 1, - "a repeated stale miss should list completion markers once" - ); - assert_eq!( - operations.delete_count(), - 0, - "a repeated withdrawal should not delete marker state" - ); assert!( - operations.inserts().is_empty(), - "a repeated stale miss should not rewrite the completed tombstone" - ); - let (stored, generation) = graph - .get(&ec_id) - .expect("should read stored tombstone") - .expect("should preserve tombstone"); - assert_eq!( - generation, 2, - "only the first withdrawal should advance the root generation" - ); - assert_eq!( - stored.consent.updated, first_updated, - "repeated withdrawal should preserve the first tombstone timestamp" + kv.clear_withdrawal_marker(&ec_id).is_err(), + "a marker that remains after delete failure should block revival" + ); + assert!( + kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) + .expect("should confirm marker remains"), + "completion marker should remain present" ); } #[test] - fn tombstone_existing_from_stale_parallel_snapshot_stops_after_conflict() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); - let ec_id = snapshot_ec_id(); - graph - .create(&ec_id, &live_entry()) - .expect("should seed live row"); - let stale_snapshot = graph.load_snapshot(&ec_id); - operations.reset(); - - let first_outcome = graph.tombstone_existing_from_snapshot(&ec_id, stale_snapshot.clone()); - let second_outcome = graph.tombstone_existing_from_snapshot(&ec_id, stale_snapshot); - - assert_eq!( - operations.inserts(), - vec![ - RecordedEcKvInsert { - mode: EcKvWriteMode::IfGenerationMatch(1), - ttl: TOMBSTONE_TTL, - }, - RecordedEcKvInsert { - mode: EcKvWriteMode::Add, - ttl: TOMBSTONE_TTL, - }, - RecordedEcKvInsert { - mode: EcKvWriteMode::IfGenerationMatch(1), - ttl: TOMBSTONE_TTL, - }, - ], - "parallel loser should attempt stale CAS once and never replace the winner" - ); + fn withdrawal_marker_is_excluded_from_hash_prefix_count() { + let hash = "a".repeat(64); + let ec_id = format!("{hash}.ABC123"); + let kv = KvIdentityGraph::in_memory("test_store"); + kv.create(&ec_id, &live_entry()) + .expect("should create live entry"); assert_eq!( - operations.lookup_count(), + kv.count_hash_prefix_keys(&hash) + .expect("should count live root"), 1, - "parallel loser should reread exactly once after its conflict" + "the root should be the only hash-prefix key" ); - assert_eq!( - second_outcome.generation_for(&ec_id), - Some(2), - "parallel loser should return the winner's stored generation" + + tombstone_via_page_load(&kv, &ec_id); + + assert!( + kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) + .expect("should read withdrawal marker"), + "withdrawal should record completion" ); assert_eq!( - second_outcome - .entry_for(&ec_id) - .map(|entry| entry.consent.updated), - first_outcome - .entry_for(&ec_id) - .map(|entry| entry.consent.updated), - "parallel loser should preserve the winner's tombstone" + kv.count_hash_prefix_keys(&hash) + .expect("should count tombstoned root"), + 1, + "the marker namespace must not affect cluster counts" ); } #[test] - fn tombstone_existing_from_snapshot_succeeds_without_backend_for_tombstone() { - let graph = KvIdentityGraph::failing("unavailable-store"); - let ec_id = snapshot_ec_id(); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(KvEntry::tombstone(1_000)), - generation: Some(3), - }; + fn delete_removes_withdrawal_marker() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + kv.create(&ec_id, &live_entry()) + .expect("should create live entry"); + tombstone_via_page_load(&kv, &ec_id); - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot.clone()); + kv.delete(&ec_id).expect("should delete entry and marker"); - assert_eq!( - outcome, snapshot, - "authoritative tombstone should not touch unavailable backend" + assert!( + kv.get(&ec_id).expect("should read store").is_none(), + "hard delete should remove the root" + ); + assert!( + !kv.withdrawal_marker_exists(&ec_id, checked_current_timestamp()) + .expect("should read withdrawal marker"), + "hard delete should remove withdrawal completion" ); } - #[test] - fn tombstone_existing_from_snapshot_non_authoritative_states_reread_live_row() { - let ec_id = snapshot_ec_id(); - let states = [ - EcKvSnapshot::NotRead, - EcKvSnapshot::Failed { - ec_id: ec_id.clone(), - }, - EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: None, - }, - EcKvSnapshot::Present { - ec_id: "different-ec-id".to_owned(), - entry: Box::new(KvEntry::tombstone(1_000)), - generation: Some(9), - }, - ]; + /// [`EcKvStore`] wrapper whose first CAS write both fails the precondition + /// and deletes the key, simulating a concurrent withdrawal that removes the + /// row between this writer's read and its write. + struct DisappearOnConflictEcKv { + inner: InMemoryEcKv, + conflicts_remaining: std::sync::Mutex, + } - for state in states { - let operations = Arc::new(RecordedEcKvOperations::default()); - let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); - graph - .create(&ec_id, &live_entry()) - .expect("should seed live row"); - operations.reset(); + impl DisappearOnConflictEcKv { + fn new(conflicts: u32) -> Self { + Self { + inner: InMemoryEcKv::new("disappear-store"), + conflicts_remaining: std::sync::Mutex::new(conflicts), + } + } - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, state); + fn seed_live(&self, ec_id: &str) { + let (body, meta) = + KvIdentityGraph::serialize_entry(&live_entry(), self.inner.store_name()) + .expect("should serialize seeded entry"); + self.inner + .insert( + ec_id, + EcKvWrite { + body: &body, + metadata: &meta, + ttl: ENTRY_TTL, + mode: EcKvWriteMode::Add, + }, + ) + .expect("should seed live entry"); + } + } - assert_eq!( - operations.lookup_count(), - 1, - "state should force one reread" - ); - assert_eq!( - operations.inserts().len(), - 2, - "live reread should write the root and completion marker" - ); - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| !entry.consent.ok), - "reread live row should be tombstoned" - ); + impl EcKvStore for DisappearOnConflictEcKv { + fn store_name(&self) -> &str { + self.inner.store_name() + } + + fn lookup(&self, key: &str) -> Result, Report> { + self.inner.lookup(key) + } + + fn key_exists(&self, key: &str) -> Result> { + self.inner.key_exists(key) + } + + fn insert( + &self, + key: &str, + write: EcKvWrite<'_>, + ) -> Result> { + if matches!(write.mode, EcKvWriteMode::IfGenerationMatch(_)) { + let mut remaining = self + .conflicts_remaining + .lock() + .expect("should lock conflict counter"); + if *remaining > 0 { + *remaining -= 1; + self.inner.delete(key).expect("should delete on conflict"); + return Ok(EcKvWriteOutcome::PreconditionFailed); + } + } + self.inner.insert(key, write) + } + + fn list_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result, Report> { + self.inner.list_keys_with_prefix(prefix, limit) + } + + fn delete(&self, key: &str) -> Result<(), Report> { + self.inner.delete(key) } } @@ -4287,64 +3760,21 @@ mod tests { let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(MAX_CAS_RETRIES, false)); let ec_id = snapshot_ec_id(); graph.create(&ec_id, &live_entry()).expect("should seed"); - let snapshot = graph.load_snapshot(&ec_id); - - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); - - assert!( - matches!(outcome, EcKvSnapshot::Failed { .. }), - "CAS exhaustion must report a failed withdrawal" - ); - let (stored, _) = graph - .get(&ec_id) - .expect("should read store") - .expect("row should remain"); - assert!( - stored.consent.ok, - "the row is still live, which is exactly why the failure must be reported" - ); - } - - #[test] - fn tombstone_existing_from_snapshot_overrides_concurrent_live_update() { - let graph = KvIdentityGraph::new(ConflictInjectingEcKv::with_partner_update_on_conflict(1)); - let ec_id = snapshot_ec_id(); - graph - .create(&ec_id, &live_entry()) - .expect("should seed live row"); - let snapshot = graph.load_snapshot(&ec_id); - - let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); - - let entry = outcome - .entry_for(&ec_id) - .expect("should return persisted tombstone"); - assert!(!entry.consent.ok, "withdrawal should win after retry"); - assert!( - entry.ids.is_empty(), - "withdrawal should clear concurrent partner IDs" - ); - } - - #[test] - fn upsert_partner_id_rejects_tombstone() { - let graph = KvIdentityGraph::in_memory("test-store"); - let ec_id = snapshot_ec_id(); - graph - .create(&ec_id, &KvEntry::tombstone(1_000)) - .expect("should seed tombstone"); + let snapshot = graph.load_snapshot(&ec_id); - let result = graph.upsert_partner_id(&ec_id, "ssp.example.com", "uid-1"); + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); - assert!(result.is_err(), "public upsert should reject a tombstone"); + assert!( + matches!(outcome, EcKvSnapshot::Failed { .. }), + "CAS exhaustion must report a failed withdrawal" + ); let (stored, _) = graph .get(&ec_id) .expect("should read store") - .expect("should preserve tombstone"); - assert!(!stored.consent.ok, "entry should remain withdrawn"); + .expect("row should remain"); assert!( - stored.ids.is_empty(), - "upsert should not repopulate partner IDs" + stored.consent.ok, + "the row is still live, which is exactly why the failure must be reported" ); } @@ -4416,444 +3846,510 @@ mod tests { } #[test] - fn key_exists_confirmed_distinguishes_absence_from_a_stale_point_read() { - let graph = KvIdentityGraph::stale_lookup("stale-store", 1); + fn stale_completion_marker_does_not_suppress_withdrawal_after_root_recreation() { + let operations = Arc::new(RecordedEcKvOperations::default()); + let graph = KvIdentityGraph::new(RecordingEcKv::with_stale_lookups( + Arc::clone(&operations), + 1, + )); let ec_id = snapshot_ec_id(); + let expired_updated = current_timestamp().saturating_sub(TOMBSTONE_TTL.as_secs() + 1); + let expired_tombstone = KvEntry::tombstone(expired_updated); + graph + .create(&ec_id, &expired_tombstone) + .expect("should seed old tombstone"); + graph + .write_withdrawal_marker(&ec_id, expired_updated) + .expect("should seed old completion marker"); + + // Model the root expiring just before its later-written completion + // marker, followed by recreation of the same key. + graph.store.delete(&ec_id).expect("should expire root row"); graph .create(&ec_id, &live_entry()) - .expect("should seed live"); + .expect("should make the same key live again after root expiry"); - assert!( - graph.get(&ec_id).expect("should read store").is_none(), - "the first point read is stale by construction" - ); - assert!( - graph - .key_exists_confirmed(&ec_id) - .expect("should list the store"), - "the list API must still see a row the point read missed" + let outcome = graph.tombstone_existing_from_snapshot( + &ec_id, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, ); - let absent = KvIdentityGraph::in_memory("empty-store"); assert!( - !absent - .key_exists_confirmed(&ec_id) - .expect("should list the store"), - "an empty store must prove absence" + outcome + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "an expired completion marker must not suppress withdrawal of a recreated live row" ); + let (stored, _) = graph + .get(&ec_id) + .expect("should read recreated row") + .expect("should retain recreated row"); + assert!(!stored.consent.ok, "recreated row should be tombstoned"); } #[test] - fn snapshot_upsert_keeps_add_confirmed_present_when_refresh_misses() { - // `generate_if_needed` records a successful `Add` as `Present` without a - // generation. EID ingestion refreshes that snapshot to obtain one; on an - // eventually-consistent store the refresh can miss. Enrichment is best - // effort, so the miss must not retract the confirmed create — otherwise - // finalization suppresses the `ts-ec` cookie for a root that was written. - let graph = KvIdentityGraph::in_memory("empty-store"); + fn tombstone_existing_from_repeated_stale_miss_preserves_first_write() { + let operations = Arc::new(RecordedEcKvOperations::default()); + let graph = KvIdentityGraph::new(RecordingEcKv::with_stale_lookups( + Arc::clone(&operations), + 2, + )); let ec_id = snapshot_ec_id(); - let add_confirmed = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: None, - }; - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, add_confirmed); + graph + .create(&ec_id, &live_entry()) + .expect("should seed live row"); + operations.reset(); - assert!( - outcome.entry_for(&ec_id).is_some(), - "an Add-confirmed row must survive a non-authoritative refresh miss" + let first_outcome = graph.tombstone_existing_from_snapshot( + &ec_id, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, ); - assert!( - graph.get(&ec_id).expect("should read store").is_none(), - "a missed refresh must not create or overwrite a root" + let first_updated = first_outcome + .entry_for(&ec_id) + .expect("should return first tombstone") + .consent + .updated; + assert_eq!( + operations.lookup_count(), + 1, + "first stale miss should retry its point read once" ); - } - - #[test] - fn snapshot_upsert_keeps_add_confirmed_present_when_refresh_fails() { - let graph = KvIdentityGraph::failing("failing-store"); - let ec_id = snapshot_ec_id(); - let add_confirmed = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: None, - }; - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, add_confirmed); - - assert!( - outcome.entry_for(&ec_id).is_some(), - "a read failure is not evidence of absence and must not retract the create" + assert_eq!( + operations.list_count(), + 1, + "first stale miss should list completion markers once" ); - } - - #[test] - fn snapshot_upsert_without_proof_still_reports_a_refresh_miss() { - // No prior proof of existence: a `NotRead` snapshot that refreshes into - // a miss must stay `Missing` so finalization can run orphan recovery. - let graph = KvIdentityGraph::in_memory("empty-store"); - let ec_id = snapshot_ec_id(); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = - graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, EcKvSnapshot::NotRead); - - assert!( - matches!(outcome, EcKvSnapshot::Missing { .. }), - "an unproven refresh miss must remain a miss" + assert_eq!( + operations.exact_check_count(), + 1, + "first stale miss should confirm root existence once" ); - } - - #[test] - fn a_store_error_never_carries_the_whole_identifier() { - // Every message in this module goes through `log_id`, so a report that - // reaches a log cannot disclose the identifier it is about. - let kv = KvIdentityGraph::failing("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - - let report = kv - .create(&ec_id, &live_entry()) - .expect_err("the failing store should error"); - - let rendered = format!("{report:?}"); - assert!( - !rendered.contains(&ec_id), - "a store error must not disclose the identifier: {rendered}" + assert_eq!( + operations.inserts().len(), + 2, + "first stale miss should write the root and completion marker" ); - } - - #[test] - fn a_locally_built_error_never_carries_the_whole_identifier() { - // The injected-failure case above covers errors the backend produces. - // These are built in this module from the identifier itself, on every - // path a request can reach: a duplicate create, single upserts naming a - // key the store does not hold or has withdrawn, and the CAS-exhaustion - // terminal errors. - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - kv.create(&ec_id, &live_entry()).expect("should create"); - - let duplicate = kv - .create(&ec_id, &live_entry()) - .expect_err("a second create should be refused"); - let missing = kv - .upsert_partner_id(&format!("{}.ZZZ999", "b".repeat(64)), "partner", "uid") - .expect_err("an upsert on a missing key should be refused"); - let withdrawn = { - assert_eq!( - kv.write_withdrawal_tombstone(&ec_id, drop) - .expect("should tombstone"), - TombstoneOutcome::Written, - "should tombstone the seeded identity" - ); - kv.upsert_partner_id(&ec_id, "partner", "uid") - .expect_err("an upsert on a withdrawn key should be refused") - }; + operations.reset(); - // The remaining CAS-exhaustion paths build their message the same way, - // and a store that never lets a write land is the only way to reach them. - let cas_revive = { - let store = ConflictInjectingEcKv::new(MAX_CAS_RETRIES + 1, false); - store.seed_tombstone(&ec_id); - KvIdentityGraph::new(store) - .create_or_revive(&ec_id, &live_entry()) - .expect_err("should exhaust CAS retries") - }; - let cas_upsert = { - let store = ConflictInjectingEcKv::new(MAX_CAS_RETRIES + 1, false); - store.seed_live(&ec_id); - KvIdentityGraph::new(store) - .upsert_partner_id(&ec_id, "partner", "uid") - .expect_err("should exhaust CAS retries") - }; - let cas_if_exists = { - let store = ConflictInjectingEcKv::new(MAX_CAS_RETRIES + 1, false); - store.seed_live(&ec_id); - KvIdentityGraph::new(store) - .upsert_partner_id_if_exists(&ec_id, "partner", "uid") - .expect_err("should exhaust CAS retries") - }; + graph.tombstone_existing_from_snapshot( + &ec_id, + EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }, + ); - for (label, report) in [ - ("duplicate create", duplicate), - ("missing key", missing), - ("withdrawn key", withdrawn), - ("CAS exhaustion reviving", cas_revive), - ("CAS exhaustion upserting", cas_upsert), - ("CAS exhaustion upserting if present", cas_if_exists), - ] { - let rendered = format!("{report:?}"); - assert!( - !rendered.contains(&ec_id) && !rendered.contains(&"b".repeat(64)), - "the {label} error must not disclose the identifier: {rendered}" - ); - } + assert_eq!( + operations.lookup_count(), + 1, + "a repeated stale miss should retry its point read once" + ); + assert_eq!( + operations.exact_check_count(), + 0, + "a valid completion marker should avoid a root existence check" + ); + assert_eq!( + operations.list_count(), + 1, + "a repeated stale miss should list completion markers once" + ); + assert_eq!( + operations.delete_count(), + 0, + "a repeated withdrawal should not delete marker state" + ); + assert!( + operations.inserts().is_empty(), + "a repeated stale miss should not rewrite the completed tombstone" + ); + let (stored, generation) = graph + .get(&ec_id) + .expect("should read stored tombstone") + .expect("should preserve tombstone"); + assert_eq!( + generation, 2, + "only the first withdrawal should advance the root generation" + ); + assert_eq!( + stored.consent.updated, first_updated, + "repeated withdrawal should preserve the first tombstone timestamp" + ); } #[test] - fn write_withdrawal_tombstone_ignores_an_identity_the_store_does_not_hold() { + fn tombstone_existing_from_snapshot_never_creates_missing_key() { let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "b".repeat(64)); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let snapshot = EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }; - assert_eq!( - kv.write_withdrawal_tombstone(&ec_id, drop) - .expect("should resolve the withdrawal"), - TombstoneOutcome::UnknownIdentity, - "an identity that was never issued has nothing to withdraw" + let outcome = kv.tombstone_existing_from_snapshot(&ec_id, snapshot); + + assert!( + matches!(outcome, EcKvSnapshot::Missing { .. }), + "withdrawal should preserve the missing snapshot for an absent identity" ); assert!( - kv.get(&ec_id).expect("should read back").is_none(), - "should not create a row for an identity the store never held" + kv.get(&ec_id).expect("should read store").is_none(), + "withdrawal must not create a tombstone for an absent key" ); } #[test] - fn withdrawing_many_unheld_identities_creates_no_rows() { - let kv = KvIdentityGraph::in_memory("test_store"); - let hash = "c".repeat(64); + fn tombstone_existing_from_snapshot_non_authoritative_states_reread_live_row() { + let ec_id = snapshot_ec_id(); + let states = [ + EcKvSnapshot::NotRead, + EcKvSnapshot::Failed { + ec_id: ec_id.clone(), + }, + EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(live_entry()), + generation: None, + }, + EcKvSnapshot::Present { + ec_id: "different-ec-id".to_owned(), + entry: Box::new(KvEntry::tombstone(1_000)), + generation: Some(9), + }, + ]; + + for state in states { + let operations = Arc::new(RecordedEcKvOperations::default()); + let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); + graph + .create(&ec_id, &live_entry()) + .expect("should seed live row"); + operations.reset(); + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, state); - // The suffix is caller-supplied, so a shared hash prefix must not be - // enough to have a row written under it. - for suffix in ["aaaaaa", "bbbbbb", "cccccc", "dddddd"] { assert_eq!( - kv.write_withdrawal_tombstone(&format!("{hash}.{suffix}"), drop) - .expect("should resolve the withdrawal"), - TombstoneOutcome::UnknownIdentity, - "suffix `{suffix}` was never issued" + operations.lookup_count(), + 1, + "state should force one reread" + ); + assert_eq!( + operations.inserts().len(), + 2, + "live reread should write the root and completion marker" + ); + assert!( + outcome + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "reread live row should be tombstoned" ); } - - assert_eq!( - kv.count_hash_prefix_keys(&hash) - .expect("should count the prefix"), - 0, - "should hold no rows under a hash nothing was issued for" - ); } #[test] - fn withdrawal_does_not_lose_a_new_identity_to_lookup_lag() { - let operations = Arc::new(RecordedEcKvOperations::default()); - let kv = KvIdentityGraph::new(RecordingEcKv::with_lagging_live_lookups(operations)); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - kv.create(&ec_id, &live_entry()) - .expect("should issue an identity"); + fn tombstone_existing_from_snapshot_overrides_concurrent_live_update() { + let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(1, true)); + let ec_id = snapshot_ec_id(); + graph + .create(&ec_id, &live_entry()) + .expect("should seed live row"); + let snapshot = graph.load_snapshot(&ec_id); + + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot); + + let entry = outcome + .entry_for(&ec_id) + .expect("should return persisted tombstone"); + assert!(!entry.consent.ok, "withdrawal should win after retry"); assert!( - kv.lookup_raw(&ec_id) - .expect("should read lagging replica") - .is_none(), - "should model the issuance replication gap" - ); - assert_eq!( - kv.write_withdrawal_tombstone(&ec_id, drop) - .expect("should process withdrawal"), - TombstoneOutcome::Written, - "should tombstone an issued identity despite a lagging lookup" - ); - assert_eq!( - kv.upsert_partner_id_if_exists(&ec_id, "partner", "uid") - .expect("should check batch-sync eligibility"), - UpsertResult::ConsentWithdrawn, - "should not revive the withdrawn identity through batch sync" + entry.ids.is_empty(), + "withdrawal should clear concurrent partner IDs" ); } #[test] - fn a_withdrawal_checks_strong_existence_once_without_eventual_lookup() { - // One existence operation may page at the backend, but must not be - // followed by an eventually consistent lookup. + fn tombstone_existing_from_snapshot_repeated_request_preserves_first_write() { let operations = Arc::new(RecordedEcKvOperations::default()); - let kv = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); + let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); + let ec_id = snapshot_ec_id(); + graph + .create(&ec_id, &live_entry()) + .expect("should seed live row"); + let live_snapshot = graph.load_snapshot(&ec_id); + operations.reset(); + + graph.tombstone_existing_from_snapshot(&ec_id, live_snapshot); - let absent = format!("{}.ABC123", "e".repeat(64)); assert_eq!( - kv.write_withdrawal_tombstone(&absent, drop) - .expect("should resolve the withdrawal"), - TombstoneOutcome::UnknownIdentity, - "an absent identity is not held" + operations.lookup_count(), + 0, + "usable generation should avoid a read" ); assert_eq!( - operations.exact_check_count(), - 1, - "an unknown identity should cost one strong read" + operations.inserts(), + vec![ + RecordedEcKvInsert { + mode: EcKvWriteMode::IfGenerationMatch(1), + ttl: TOMBSTONE_TTL, + }, + RecordedEcKvInsert { + mode: EcKvWriteMode::Add, + ttl: TOMBSTONE_TTL, + }, + ], + "first withdrawal should write the root and its completion marker" ); + let first_snapshot = graph.load_snapshot(&ec_id); + let (first_entry, first_generation) = match &first_snapshot { + EcKvSnapshot::Present { + entry, generation, .. + } => (entry.as_ref().clone(), *generation), + other => panic!("should load first tombstone, got {other:?}"), + }; + operations.reset(); + + let second_outcome = graph.tombstone_existing_from_snapshot(&ec_id, first_snapshot); - let held = format!("{}.ABC123", "a".repeat(64)); - kv.create(&held, &live_entry()).expect("should create"); - let before = operations.exact_check_count(); assert_eq!( - kv.write_withdrawal_tombstone(&held, drop) - .expect("should resolve the withdrawal"), - TombstoneOutcome::Written, - "a held identity is tombstoned" + operations.operation_count(), + 0, + "repeated withdrawal should not access the backend" ); assert_eq!( - operations.exact_check_count() - before, - 1, - "a held identity is checked once, then written" + second_outcome.generation_for(&ec_id), + first_generation, + "repeated withdrawal should preserve the stored generation" ); assert_eq!( - operations.lookup_count(), - 0, - "should not use an eventual lookup for withdrawal" + second_outcome + .entry_for(&ec_id) + .map(|entry| entry.consent.updated), + Some(first_entry.consent.updated), + "repeated withdrawal should preserve the first tombstone timestamp" ); } #[test] - fn write_withdrawal_tombstone_refuses_an_empty_identifier() { - let kv = KvIdentityGraph::in_memory("test_store"); - kv.create(&format!("{}.ABC123", "f".repeat(64)), &live_entry()) - .expect("should create"); - - assert_eq!( - kv.write_withdrawal_tombstone("", drop) - .expect("should resolve the withdrawal"), - TombstoneOutcome::UnknownIdentity, - "an empty identifier names no key and must not withdraw anything" - ); - let (held, _) = kv - .get(&format!("{}.ABC123", "f".repeat(64))) - .expect("should read back") - .expect("should still hold the identity"); - assert!( - held.consent.ok, - "should not have withdrawn an unrelated row" - ); - } - - /// Store double whose reads always fail while writes still work. - struct ReadFailingEcKv { - inner: super::super::kv_backend::test_support::InMemoryEcKv, - } - - impl ReadFailingEcKv { - fn new() -> Self { - Self { - inner: super::super::kv_backend::test_support::InMemoryEcKv::new("test_store"), - } - } - } - - impl EcKvStore for ReadFailingEcKv { - fn store_name(&self) -> &str { - self.inner.store_name() - } + fn tombstone_existing_from_snapshot_skips_backend_for_authoritative_tombstone() { + for generation in [Some(7), None] { + let operations = Arc::new(RecordedEcKvOperations::default()); + let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); + let ec_id = snapshot_ec_id(); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(KvEntry::tombstone(1_000)), + generation, + }; - fn lookup(&self, _key: &str) -> Result, Report> { - Err(Report::new(TrustedServerError::KvStore { - store_name: "test_store".to_owned(), - message: "reads unavailable".to_owned(), - })) - } + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot.clone()); - fn key_exists(&self, key: &str) -> Result> { - self.lookup(key).map(|entry| entry.is_some()) + assert_eq!( + outcome, snapshot, + "should preserve authoritative tombstone state" + ); + assert_eq!( + operations.operation_count(), + 0, + "an authoritative tombstone should not access the backend" + ); } + } - fn insert( - &self, - key: &str, - write: EcKvWrite<'_>, - ) -> Result> { - self.inner.insert(key, write) - } + #[test] + fn tombstone_existing_from_snapshot_succeeds_without_backend_for_tombstone() { + let graph = KvIdentityGraph::failing("unavailable-store"); + let ec_id = snapshot_ec_id(); + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(KvEntry::tombstone(1_000)), + generation: Some(3), + }; - // Left working so a test can prove no row was created without going - // through the read path it just made fail. - fn list_keys_with_prefix( - &self, - prefix: &str, - limit: u32, - ) -> Result, Report> { - self.inner.list_keys_with_prefix(prefix, limit) - } + let outcome = graph.tombstone_existing_from_snapshot(&ec_id, snapshot.clone()); - fn delete(&self, key: &str) -> Result<(), Report> { - self.inner.delete(key) - } + assert_eq!( + outcome, snapshot, + "authoritative tombstone should not touch unavailable backend" + ); } #[test] - fn a_failing_check_is_not_a_way_to_write_for_an_identity_that_was_never_issued() { - // The caller controls the identifier and can drive load, so a store - // failure must not become a route to the write this gate exists to - // prevent. - let kv = KvIdentityGraph::new(ReadFailingEcKv::new()); - let hash = "8".repeat(64); - let ec_id = format!("{hash}.ABC123"); + fn tombstone_existing_from_snapshot_uses_existing_generation() { + let kv = KvIdentityGraph::in_memory("test_store"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + kv.create(&ec_id, &live_entry()).expect("should create"); + let snapshot = kv.load_snapshot(&ec_id); + + let outcome = kv.tombstone_existing_from_snapshot(&ec_id, snapshot); assert!( - kv.write_withdrawal_tombstone(&ec_id, drop).is_err(), - "a check that cannot answer is a fault, so a caller inspecting only \ - the error case still reports it" - ); - assert_eq!( - kv.count_hash_prefix_keys(&hash) - .expect("should count the prefix"), - 0, - "should not create a row while the store is degraded" + outcome + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "should return the persisted tombstone" ); + let (stored, _) = kv + .get(&ec_id) + .expect("should read store") + .expect("should preserve existing key"); + assert!(!stored.consent.ok, "should persist withdrawal state"); } #[test] - fn a_caller_that_only_inspects_the_error_case_still_sees_a_failed_check() { - // The withdrawal call site is edited by more than one branch. Reporting - // a failed check through `Err` means the common - // `if let Err(..) = ...` shape cannot discard it, where a third `Ok` - // variant would be dropped without a compiler complaint. - let kv = KvIdentityGraph::new(ReadFailingEcKv::new()); - let ec_id = format!("{}.ABC123", "6".repeat(64)); + fn tombstone_existing_from_stale_parallel_snapshot_stops_after_conflict() { + let operations = Arc::new(RecordedEcKvOperations::default()); + let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); + let ec_id = snapshot_ec_id(); + graph + .create(&ec_id, &live_entry()) + .expect("should seed live row"); + let stale_snapshot = graph.load_snapshot(&ec_id); + operations.reset(); - let mut reported = false; - if let Err(_err) = kv.write_withdrawal_tombstone(&ec_id, drop) { - reported = true; - } + let first_outcome = graph.tombstone_existing_from_snapshot(&ec_id, stale_snapshot.clone()); + let second_outcome = graph.tombstone_existing_from_snapshot(&ec_id, stale_snapshot); - assert!(reported, "a failed check must reach an error-only caller"); + assert_eq!( + operations.inserts(), + vec![ + RecordedEcKvInsert { + mode: EcKvWriteMode::IfGenerationMatch(1), + ttl: TOMBSTONE_TTL, + }, + RecordedEcKvInsert { + mode: EcKvWriteMode::Add, + ttl: TOMBSTONE_TTL, + }, + RecordedEcKvInsert { + mode: EcKvWriteMode::IfGenerationMatch(1), + ttl: TOMBSTONE_TTL, + }, + ], + "parallel loser should attempt stale CAS once and never replace the winner" + ); + assert_eq!( + operations.lookup_count(), + 1, + "parallel loser should reread exactly once after its conflict" + ); + assert_eq!( + second_outcome.generation_for(&ec_id), + Some(2), + "parallel loser should return the winner's stored generation" + ); + assert_eq!( + second_outcome + .entry_for(&ec_id) + .map(|entry| entry.consent.updated), + first_outcome + .entry_for(&ec_id) + .map(|entry| entry.consent.updated), + "parallel loser should preserve the winner's tombstone" + ); } #[test] - fn a_longer_key_does_not_answer_for_the_identity_it_starts_with() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "7".repeat(64)); - // Only a longer key exists. The identity itself was never issued, so a - // check that matched by prefix would report it as held and tombstone it. - kv.create(&format!("{ec_id}trailing"), &live_entry()) - .expect("should create"); + fn withdrawal_failed_clock_does_not_create_absent_root() { + let operations = Arc::new(RecordedEcKvOperations::default()); + let graph = KvIdentityGraph::new(RecordingEcKv::new(Arc::clone(&operations))); + let ec_id = snapshot_ec_id(); + graph + .write_withdrawal_marker(&ec_id, 1000) + .expect("should seed a marker without a root"); + operations.reset(); + let missing = graph.load_snapshot(&ec_id); + + let outcome = graph.tombstone_unproven_missing(&ec_id, missing, None); assert!( - !kv.key_exists_confirmed(&ec_id).expect("should check"), - "a longer key is a different identity" + matches!(outcome, EcKvSnapshot::Missing { .. }), + "absent root should remain missing" + ); + assert!( + graph + .get(&ec_id) + .expect("should read absent root") + .is_none(), + "clock failure must not mint an unknown root" ); assert_eq!( - kv.write_withdrawal_tombstone(&ec_id, drop) - .expect("should resolve the withdrawal"), - TombstoneOutcome::UnknownIdentity, - "should not tombstone an identity the store never held" + operations.exact_check_count(), + 1, + "should prove root absence despite the marker" ); assert!( - kv.get(&ec_id).expect("should read back").is_none(), - "should not create a row via a prefix match" + operations.inserts().is_empty(), + "absent identity should cause no writes" ); } #[test] - fn key_exists_confirmed_distinguishes_held_identities() { - let kv = KvIdentityGraph::in_memory("test_store"); - let held = format!("{}.ABC123", "d".repeat(64)); - let sibling = format!("{}.ZZZ999", "d".repeat(64)); - kv.create(&held, &live_entry()).expect("should create"); + fn withdrawal_failed_clock_stale_miss_tombstones_live_root() { + let operations = Arc::new(RecordedEcKvOperations::default()); + let graph = KvIdentityGraph::new(RecordingEcKv::with_stale_lookups( + Arc::clone(&operations), + 1, + )); + let ec_id = snapshot_ec_id(); + graph + .create(&ec_id, &concurrent_live_entry()) + .expect("should seed a live row with partner IDs"); + graph + .write_withdrawal_marker(&ec_id, 1000) + .expect("should seed a completion marker left after root recreation"); + operations.reset(); + let missing = graph.load_snapshot(&ec_id); + assert!( + matches!(missing, EcKvSnapshot::Missing { .. }), + "should reproduce a stale point-read miss for the live row" + ); + // None is the checked clock's failure result, not Unix epoch zero. + let outcome = graph.tombstone_unproven_missing(&ec_id, missing, None); + + let (stored, generation) = graph + .get(&ec_id) + .expect("should read back persisted withdrawal") + .expect("should retain the root"); assert!( - kv.key_exists_confirmed(&held).expect("should check"), - "should confirm a held identity" + !stored.consent.ok, + "an unusable clock must not let a marker suppress withdrawal" ); assert!( - !kv.key_exists_confirmed(&sibling).expect("should check"), - "a different suffix under the same hash is a different identity" + stored.ids.is_empty(), + "withdrawal should clear stored partner IDs" + ); + assert_eq!( + generation, 2, + "withdrawal should write the existing root once" + ); + assert!( + outcome + .entry_for(&ec_id) + .is_some_and(|entry| !entry.consent.ok), + "clock failure should not fail a strongly confirmed withdrawal" + ); + assert_eq!( + operations.exact_check_count(), + 1, + "should strongly check the root" + ); + assert_eq!( + operations.inserts(), + vec![ + RecordedEcKvInsert { + mode: EcKvWriteMode::Overwrite, + ttl: TOMBSTONE_TTL + }, + RecordedEcKvInsert { + mode: EcKvWriteMode::Add, + ttl: TOMBSTONE_TTL + }, + ], + "should write only the root tombstone and its completion marker" ); } } diff --git a/crates/trusted-server-core/src/ec/kv_backend.rs b/crates/trusted-server-core/src/ec/kv_backend.rs index bfe74ca6f3..70a999ffa0 100644 --- a/crates/trusted-server-core/src/ec/kv_backend.rs +++ b/crates/trusted-server-core/src/ec/kv_backend.rs @@ -178,6 +178,68 @@ pub(crate) mod test_support { use super::*; + /// [`EcKvStore`] whose reads succeed but every write fails, simulating a + /// store that becomes unwritable mid-request. + pub(crate) struct WriteFailingEcKv { + inner: InMemoryEcKv, + } + + impl WriteFailingEcKv { + pub(crate) fn new() -> Self { + Self { + inner: InMemoryEcKv::new("write-failing-store"), + } + } + + /// Stores a row directly, bypassing the failing write path. + pub(crate) fn seed(&self, key: &str, body: &str, metadata: &str) { + self.inner + .insert( + key, + EcKvWrite { + body, + metadata, + ttl: Duration::from_secs(60), + mode: EcKvWriteMode::Overwrite, + }, + ) + .expect("should seed the in-memory store"); + } + } + + impl EcKvStore for WriteFailingEcKv { + fn store_name(&self) -> &str { + self.inner.store_name() + } + fn lookup(&self, key: &str) -> Result, Report> { + self.inner.lookup(key) + } + fn key_exists(&self, key: &str) -> Result> { + self.inner.key_exists(key) + } + + fn insert( + &self, + _key: &str, + _write: EcKvWrite<'_>, + ) -> Result> { + Err(Report::new(TrustedServerError::KvStore { + store_name: self.inner.store_name().to_owned(), + message: "write failing test store".to_owned(), + })) + } + fn list_keys_with_prefix( + &self, + prefix: &str, + limit: u32, + ) -> Result, Report> { + self.inner.list_keys_with_prefix(prefix, limit) + } + fn delete(&self, key: &str) -> Result<(), Report> { + self.inner.delete(key) + } + } + /// [`EcKvStore`] wrapper that counts `lookup` calls through a shared counter /// so tests can prove exactly how many reads a flow performs. pub(crate) struct CountingEcKv { diff --git a/crates/trusted-server-core/src/ec/kv_types.rs b/crates/trusted-server-core/src/ec/kv_types.rs index 589f280c11..3772f07f3c 100644 --- a/crates/trusted-server-core/src/ec/kv_types.rs +++ b/crates/trusted-server-core/src/ec/kv_types.rs @@ -6,10 +6,16 @@ //! //! The schema is versioned (`v: 1`) to allow future migrations. +use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet}; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize, de}; +use serde_json::value::RawValue; +use super::eid_types::{ + AgentType, EidFields, Expiry, MAX_EID_FIELD_BYTES, MatchMethod, ProviderId, RawJson, UidValue, + UnixSeconds, optional_exp, +}; use crate::consent::ConsentContext; use crate::geo::GeoInfo; @@ -30,14 +36,27 @@ pub const MAX_SEEN_DOMAINS: usize = 50; pub const MAX_STORED_DOMAIN_LENGTH: usize = 255; /// Maximum allowed length (in bytes) for a partner UID across all sync -/// mechanisms (pixel, batch, pull). Defined centrally to ensure consistent -/// validation. +/// mechanisms (pixel, batch, pull) and browser ingest. Defined centrally to +/// ensure consistent validation. +/// +/// Builds before identity providers reject a row holding a longer UID, so +/// these writers stay at this limit until rollback past them is no longer +/// supported; only provider writes use [`MAX_STORED_UID_LENGTH`]. pub const MAX_UID_LENGTH: usize = 512; +/// Maximum length (in bytes) of a UID a row may hold: the limit for identity +/// provider writes and for reading stored rows. Sized for `LiveRamp` envelopes +/// (about 700 bytes). +pub const MAX_STORED_UID_LENGTH: usize = 1024; + +/// How long a provider opt-out keeps that provider from being called again, +/// in seconds (30 days). +pub const OPT_OUT_MEMORY_SECONDS: u64 = 30 * 24 * 60 * 60; + /// Full KV entry stored as the body of an EC identity graph record. /// /// **KV key:** Full EC ID (`{64hex}.{6alnum}`). -/// **KV value:** JSON-serialized `KvEntry` (max ~5KB). +/// **KV value:** JSON-serialized `KvEntry` (at most [`MAX_ENTRY_BYTES`]). #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct KvEntry { /// Schema version — always [`SCHEMA_VERSION`]. @@ -62,6 +81,10 @@ pub struct KvEntry { /// Populated by pixel sync, batch sync, and pull sync operations. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub ids: BTreeMap, + /// Provider ID → Unix time until which that provider's opt-out is + /// honoured without calling the provider again. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub opt_outs: BTreeMap, } /// Consent state within a KV entry. @@ -97,11 +120,364 @@ pub struct KvGeo { pub dma: Option, } -/// A partner user ID within a KV entry. +/// Maximum serialized size of a KV entry body. Every writer refuses a larger +/// row; only the page-load planner trims entries to fit. +pub const MAX_ENTRY_BYTES: usize = 16 * 1024; + +/// A partner ID record within a KV entry, keyed by source domain. +/// +/// `uid` is always present so older readers keep working. Records written by +/// an identity provider also carry full-fidelity `eids`; new code reads +/// `eids` whenever it is non-empty and treats `uid` as derived. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct KvPartnerId { - /// The partner's user identifier. + /// First live UID of `eids`, or the single UID for records written by + /// browser ingest, batch sync or older versions. pub uid: String, + /// Full-fidelity entries written by a provider. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + eids: Vec, +} + +impl KvPartnerId { + /// Builds a single-UID record (browser ingest, batch sync, pull sync). + #[must_use] + pub fn single(uid: impl Into) -> Self { + Self { + uid: uid.into(), + eids: Vec::new(), + } + } + + /// Builds a provider record, keeping only entries with live UIDs. + /// + /// Returns `None` when no live UID remains. + #[must_use] + pub fn from_eids(eids: Vec, now: UnixSeconds) -> Option { + let mut record = Self { + uid: String::new(), + eids, + }; + record.prune_expired(now); + (!record.eids.is_empty()).then_some(record) + } + + /// Full-fidelity provider entries; empty for single-UID records. + #[must_use] + pub fn eids(&self) -> &[KvEid] { + &self.eids + } + + /// Whether this record was written by an identity provider. + #[must_use] + pub fn is_provider_written(&self) -> bool { + !self.eids.is_empty() + } + + /// Whether a UID `owner` wrote asks it for a refresh by `now`. + /// + /// UIDs another provider wrote never make the record refresh-due for + /// `owner`, so their refresh state is never handed to it. + #[must_use] + pub fn refresh_due_for(&self, owner: &ProviderId, now: UnixSeconds) -> bool { + self.eids + .iter() + .flat_map(KvEid::uids) + .any(|uid| uid.owner() == Some(owner) && uid.refresh_due(now)) + } + + /// Whether the record holds a UID that may be served at `now`. + #[must_use] + pub fn has_live_uid(&self, now: UnixSeconds) -> bool { + if self.eids.is_empty() { + return !self.uid.is_empty(); + } + self.eids + .iter() + .flat_map(KvEid::uids) + .any(|uid| uid.is_live(now)) + } + + /// Drops expired UIDs and empty entries, then resyncs `uid`. + /// + /// Returns whether anything changed. Single-UID records never change. + pub fn prune_expired(&mut self, now: UnixSeconds) -> bool { + if self.eids.is_empty() && !self.uid.is_empty() { + return false; + } + let mut changed = false; + for eid in &mut self.eids { + let held = eid.uids.len(); + eid.uids.retain(|uid| uid.is_live(now)); + changed |= eid.uids.len() != held; + } + self.eids.retain(|eid| !eid.uids.is_empty()); + let first = self + .eids + .first() + .and_then(|eid| eid.uids.first()) + .map_or("", |uid| uid.id().as_str()); + if self.uid != first { + self.uid = first.to_owned(); + changed = true; + } + changed + } + + /// Drops entries left without UIDs and resyncs `uid`. + /// + /// Returns whether the record still holds a UID within + /// [`MAX_STORED_UID_LENGTH`]. + fn retain_readable(&mut self) -> bool { + if self.eids.is_empty() { + return self.uid.len() <= MAX_STORED_UID_LENGTH; + } + self.eids.retain(|eid| !eid.uids.is_empty()); + self.uid = self + .eids + .first() + .and_then(|eid| eid.uids.first()) + .map(|uid| uid.id().as_str().to_owned()) + .unwrap_or_default(); + !self.eids.is_empty() + } + + /// Removes the last UID of the last entry and resyncs `uid`; used by the + /// write planner's size cap. + pub(crate) fn drop_last_uid(&mut self) { + if let Some(eid) = self.eids.last_mut() { + eid.uids.pop(); + } + self.eids.retain(|eid| !eid.uids.is_empty()); + self.uid = self + .eids + .first() + .and_then(|eid| eid.uids.first()) + .map(|uid| uid.id().as_str().to_owned()) + .unwrap_or_default(); + } +} + +/// One stored EID entry with its EID-level fields. +/// +/// Stored flat (`{"inserter": .., "uids": [..]}`). Serialization flattens +/// [`EidFields`]; deserialization goes through a flat private representation +/// because serde's `flatten` cannot deserialize the `RawValue` behind `ext`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(from = "KvEidRepr")] +pub struct KvEid { + #[serde(flatten)] + fields: EidFields, + uids: Vec, +} + +/// Flat stored shape of [`KvEid`], used only to deserialize it. +#[derive(Deserialize)] +struct KvEidRepr { + #[serde(default)] + inserter: Option>, + #[serde(default)] + matcher: Option>, + #[serde(default)] + mm: Option, + #[serde(default)] + ext: Option, + #[serde(deserialize_with = "deserialize_readable_uids")] + uids: Vec, +} + +/// The `id` of a stored UID, read without validating the rest of it. +#[derive(Deserialize)] +struct StoredId<'a> { + #[serde(borrow)] + id: Cow<'a, str>, +} + +/// Deserializes stored UIDs, skipping any whose `id` exceeds +/// [`MAX_STORED_UID_LENGTH`]. +/// +/// A row written under a higher limit then still loads; an entry left +/// without UIDs is removed by [`KvEntry::drop_unreadable_ids`]. Any other +/// invalid UID still fails the row. +fn deserialize_readable_uids<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + let raw = Vec::>::deserialize(deserializer)?; + let mut uids = Vec::with_capacity(raw.len()); + for value in &raw { + let oversized = serde_json::from_str::>(value.get()) + .is_ok_and(|stored| stored.id.len() > MAX_STORED_UID_LENGTH); + if oversized { + continue; + } + uids.push(serde_json::from_str(value.get()).map_err(de::Error::custom)?); + } + Ok(uids) +} + +impl From for KvEid { + fn from(repr: KvEidRepr) -> Self { + let KvEidRepr { + inserter, + matcher, + mm, + ext, + uids, + } = repr; + Self { + fields: EidFields { + inserter, + matcher, + mm, + ext, + }, + uids, + } + } +} + +impl KvEid { + /// Builds an entry from validated parts. + #[must_use] + pub fn new(fields: EidFields, uids: Vec) -> Self { + Self { fields, uids } + } + + /// EID-level fields. + #[must_use] + pub fn fields(&self) -> &EidFields { + &self.fields + } + + /// Stored UIDs, live or not. + #[must_use] + pub fn uids(&self) -> &[KvUid] { + &self.uids + } +} + +/// One stored UID with its owner and expiry. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct KvUid { + id: UidValue, + atype: AgentType, + #[serde(default, skip_serializing_if = "Option::is_none")] + ext: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + owner: Option, + #[serde( + default, + rename = "exp", + with = "optional_exp", + skip_serializing_if = "Expiry::is_never" + )] + expiry: Expiry, + /// Private refresh state; never served, mirrored or forwarded. + #[serde(default, skip_serializing_if = "Option::is_none")] + refresh: Option, +} + +/// Refresh state a provider keeps with one of its UIDs. +/// +/// It stays in the row only: outbound EIDs are built from the UID's `id`, +/// `atype` and `ext`, so this state never reaches bidders or cookies. +#[derive(Clone, PartialEq, Serialize, Deserialize)] +pub struct KvRefresh { + /// Provider-defined state, opaque to the framework. + pub state: String, + /// When the owning provider should be asked to refresh the UID. + pub from: UnixSeconds, +} + +impl core::fmt::Debug for KvRefresh { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("KvRefresh") + .field("from", &self.from) + .finish_non_exhaustive() + } +} + +impl KvUid { + /// Builds a UID from validated parts. + #[must_use] + pub fn new( + id: UidValue, + atype: AgentType, + ext: Option, + owner: Option, + expiry: Expiry, + ) -> Self { + Self { + id, + atype, + ext, + owner, + expiry, + refresh: None, + } + } + + /// Attaches the owning provider's private refresh state. + #[must_use] + pub fn with_refresh(mut self, refresh: KvRefresh) -> Self { + self.refresh = Some(refresh); + self + } + + /// The owning provider's private refresh state. + #[must_use] + pub fn refresh(&self) -> Option<&KvRefresh> { + self.refresh.as_ref() + } + + /// Whether the owning provider asked to refresh this live UID by `now`. + /// + /// An expired UID is never refresh-due: its source is due anyway, and a + /// refresh that keeps failing must not outlive the UID. + #[must_use] + pub fn refresh_due(&self, now: UnixSeconds) -> bool { + self.is_live(now) + && self + .refresh + .as_ref() + .is_some_and(|refresh| refresh.from <= now) + } + + /// The UID value. + #[must_use] + pub fn id(&self) -> &UidValue { + &self.id + } + + /// The agent type. + #[must_use] + pub fn atype(&self) -> AgentType { + self.atype + } + + /// The UID-level `ext`. + #[must_use] + pub fn ext(&self) -> Option<&RawJson> { + self.ext.as_ref() + } + + /// The provider that wrote this UID. + #[must_use] + pub fn owner(&self) -> Option<&ProviderId> { + self.owner.as_ref() + } + + /// When the UID stops being served. + #[must_use] + pub fn expiry(&self) -> Expiry { + self.expiry + } + + /// Whether the UID may be served at `now`. + #[must_use] + pub fn is_live(&self, now: UnixSeconds) -> bool { + self.expiry.is_live(now) + } } /// Publisher property metadata captured when an EC entry is created. @@ -263,6 +639,17 @@ pub(crate) fn validated_stored_domain(domain: &str) -> Option { } impl KvEntry { + /// Removes partner records holding no UID this build can serve. + /// + /// Lets this build load rows written by one with a higher + /// [`MAX_STORED_UID_LENGTH`] instead of rejecting the whole row. Returns the + /// number of records removed. + pub fn drop_unreadable_ids(&mut self) -> usize { + let before = self.ids.len(); + self.ids.retain(|_, record| record.retain_readable()); + before - self.ids.len() + } + /// Creates a new live entry from the current request context. /// /// `domain` is the publisher's apex domain (e.g. `"example.com"`), @@ -293,7 +680,35 @@ impl KvEntry { device: None, network: None, ids: BTreeMap::new(), + opt_outs: BTreeMap::new(), + } + } + + /// Whether `provider` opted out and must not be called again by `now`. + #[must_use] + pub fn opted_out(&self, provider: &ProviderId, now: UnixSeconds) -> bool { + self.opt_outs + .get(provider) + .is_some_and(|until| *until > now.get()) + } + + /// Remembers an opt-out by `provider` for [`OPT_OUT_MEMORY_SECONDS`]. + /// + /// An opt-out already remembered keeps its end time, so a provider skipped + /// because of it is asked again once the time passes. + pub fn remember_opt_out(&mut self, provider: &ProviderId, now: UnixSeconds) { + if self.opted_out(provider, now) { + return; } + self.opt_outs.insert( + provider.clone(), + now.saturating_add_secs(OPT_OUT_MEMORY_SECONDS).get(), + ); + } + + /// Forgets opt-outs whose time has passed. + pub fn prune_opt_outs(&mut self, now: UnixSeconds) { + self.opt_outs.retain(|_, until| *until > now.get()); } /// Creates a minimal live entry. @@ -303,12 +718,7 @@ impl KvEntry { #[must_use] pub fn minimal(partner_id: &str, uid: &str, now: u64) -> Self { let mut ids = BTreeMap::new(); - ids.insert( - partner_id.to_owned(), - KvPartnerId { - uid: uid.to_owned(), - }, - ); + ids.insert(partner_id.to_owned(), KvPartnerId::single(uid.to_owned())); Self { v: SCHEMA_VERSION, created: now, @@ -328,6 +738,7 @@ impl KvEntry { device: None, network: None, ids, + opt_outs: BTreeMap::new(), } } @@ -360,6 +771,7 @@ impl KvEntry { device: None, network: None, ids: BTreeMap::new(), + opt_outs: BTreeMap::new(), } } @@ -381,12 +793,29 @@ impl KvEntry { } for (partner_id, partner_uid) in &self.ids { - if partner_uid.uid.len() > MAX_UID_LENGTH { + if partner_uid.uid.len() > MAX_STORED_UID_LENGTH { return Err(format!( - "partner ID '{partner_id}' exceeds MAX_UID_LENGTH ({})", + "partner ID '{partner_id}' exceeds MAX_STORED_UID_LENGTH ({})", partner_uid.uid.len() )); } + let oversized_field = partner_uid.eids().iter().any(|eid| { + let fields = eid.fields(); + [fields.inserter.as_deref(), fields.matcher.as_deref()] + .into_iter() + .flatten() + .any(|field| field.len() > MAX_EID_FIELD_BYTES) + }); + if oversized_field { + return Err(format!( + "partner ID '{partner_id}' has an EID inserter or matcher over MAX_EID_FIELD_BYTES" + )); + } + if partner_uid.eids().iter().any(|eid| eid.uids().is_empty()) { + return Err(format!( + "partner ID '{partner_id}' has an EID entry with no UIDs" + )); + } } if let Some(pub_properties) = &self.pub_properties { @@ -463,6 +892,9 @@ impl KvGeo { #[cfg(test)] mod tests { use super::*; + use crate::ec::eid_types::{ + AgentType, EidFields, Expiry, MatchMethod, ProviderId, RawJson, UidValue, UnixSeconds, + }; fn sample_consent_context() -> ConsentContext { ConsentContext { @@ -492,9 +924,7 @@ mod tests { let mut entry = KvEntry::new(&consent, Some(&geo), 1_741_824_000, "example.com"); entry.ids.insert( "liveramp".to_owned(), - KvPartnerId { - uid: "LR_xyz".to_owned(), - }, + KvPartnerId::single("LR_xyz".to_owned()), ); let json = serde_json::to_string(&entry).expect("should serialize KvEntry"); @@ -737,16 +1167,14 @@ mod tests { let mut entry = KvEntry::tombstone(1000); entry.ids.insert( "ssp_x".to_owned(), - KvPartnerId { - uid: "x".repeat(MAX_UID_LENGTH + 1), - }, + KvPartnerId::single("x".repeat(MAX_STORED_UID_LENGTH + 1)), ); let err = entry .validate() .expect_err("should reject oversized partner UIDs"); assert!( - err.contains("MAX_UID_LENGTH"), + err.contains("MAX_STORED_UID_LENGTH"), "should describe the UID length validation failure" ); } @@ -1206,4 +1634,244 @@ mod tests { "known_browser should default to None" ); } + + fn provider_uid(id: &str, expiry: Expiry) -> KvUid { + KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + expiry, + ) + } + + #[test] + fn legacy_partner_id_round_trips_without_eids() { + let legacy: KvPartnerId = serde_json::from_value(serde_json::json!({ "uid": "abc" })) + .expect("should read legacy record"); + assert_eq!(legacy, KvPartnerId::single("abc"), "should read legacy uid"); + assert_eq!( + serde_json::to_value(&legacy).expect("should serialize"), + serde_json::json!({ "uid": "abc" }), + "should not add eids to legacy records" + ); + } + + #[test] + fn from_eids_sets_uid_to_first_live_uid() { + let now = UnixSeconds::new(100); + let eid = KvEid::new( + EidFields::default(), + vec![ + provider_uid("expired", Expiry::At(UnixSeconds::new(50))), + provider_uid("live", Expiry::Never), + ], + ); + let record = KvPartnerId::from_eids(vec![eid], now).expect("should keep live uid"); + assert_eq!(record.uid, "live", "should mirror first live uid into uid"); + assert!(record.is_provider_written(), "should mark provider record"); + } + + #[test] + fn prune_expired_drops_dead_uids_and_resyncs_uid() { + let eid = KvEid::new( + EidFields::default(), + vec![ + provider_uid("first", Expiry::At(UnixSeconds::new(150))), + provider_uid("second", Expiry::Never), + ], + ); + let mut record = + KvPartnerId::from_eids(vec![eid], UnixSeconds::new(100)).expect("should build"); + assert_eq!(record.uid, "first", "should start on first uid"); + + let changed = record.prune_expired(UnixSeconds::new(200)); + + assert!(changed, "should report pruning"); + assert_eq!(record.eids()[0].uids().len(), 1, "should drop expired uid"); + assert_eq!(record.uid, "second", "should resync uid"); + } + + #[test] + fn provider_record_round_trips_with_owner_and_exp() { + let eid = KvEid::new( + EidFields::default(), + vec![provider_uid("v", Expiry::At(UnixSeconds::new(500)))], + ); + let record = KvPartnerId::from_eids(vec![eid], UnixSeconds::new(1)).expect("should build"); + let json = serde_json::to_value(&record).expect("should serialize"); + assert_eq!( + json, + serde_json::json!({ "uid": "v", "eids": [{ "uids": [{ "id": "v", "atype": 1, "owner": "example-provider", "exp": 500 }] }] }), + "should store owner and exp" + ); + let back: KvPartnerId = serde_json::from_value(json).expect("should deserialize"); + assert_eq!(back, record, "should round-trip"); + } + + #[test] + fn validate_rejects_empty_provider_eid() { + let mut entry = KvEntry::minimal("ids.example.com", "v", 1); + let record: KvPartnerId = + serde_json::from_value(serde_json::json!({ "uid": "v", "eids": [{ "uids": [] }] })) + .expect("should parse"); + entry.ids.insert("ids.example.com".to_owned(), record); + assert!(entry.validate().is_err(), "should reject EID with no UIDs"); + } + + #[test] + fn validate_accepts_liveramp_sized_uid() { + let entry = KvEntry::minimal("liveramp.example.com", &"x".repeat(703), 1); + assert!( + entry.validate().is_ok(), + "should accept a 703-byte envelope-sized UID" + ); + } + + #[test] + fn drop_unreadable_ids_removes_oversized_single_uid() { + let mut entry = KvEntry::minimal("ids.example.com", "kept", 1); + entry.ids.insert( + "long.example.com".to_owned(), + KvPartnerId::single("x".repeat(MAX_STORED_UID_LENGTH + 1)), + ); + + let dropped = entry.drop_unreadable_ids(); + + assert_eq!(dropped, 1, "should drop the oversized record"); + assert_eq!( + entry.ids.keys().collect::>(), + vec!["ids.example.com"], + "should keep the readable record" + ); + assert!(entry.validate().is_ok(), "should validate after dropping"); + } + + #[test] + fn deserialize_drops_oversized_provider_uids() { + let long = "x".repeat(MAX_STORED_UID_LENGTH + 1); + let json = serde_json::json!({ + "v": SCHEMA_VERSION, + "created": 1, + "consent": { "ok": true, "updated": 1 }, + "geo": { "country": "US" }, + "ids": { + "partial.example.com": { "uid": long, "eids": [ + { "uids": [{ "id": long, "atype": 1 }] }, + { "uids": [{ "id": "short", "atype": 1 }] } + ] }, + "gone.example.com": { "uid": long, "eids": [ + { "uids": [{ "id": long, "atype": 1 }] } + ] } + } + }) + .to_string(); + let mut entry: KvEntry = + serde_json::from_str(&json).expect("should parse a row with oversized UIDs"); + + let dropped = entry.drop_unreadable_ids(); + + assert_eq!(dropped, 1, "should drop the record left without UIDs"); + let partial = entry + .ids + .get("partial.example.com") + .expect("should keep the record with a readable UID"); + assert_eq!(partial.uid, "short", "should resync uid to the kept UID"); + assert_eq!(partial.eids().len(), 1, "should drop the emptied entry"); + assert!(entry.validate().is_ok(), "should validate after dropping"); + } + + #[test] + fn provider_record_round_trips_eid_level_fields() { + let ext = serde_json::value::to_raw_value(&serde_json::json!({ "k": 1 })) + .expect("should encode ext"); + let fields = EidFields { + inserter: Some("inserter.example.com".into()), + matcher: Some("matcher.example.com".into()), + mm: Some(MatchMethod::Authenticated), + ext: Some(RawJson::try_from(ext.as_ref()).expect("should accept ext")), + }; + let eid = KvEid::new(fields, vec![provider_uid("v", Expiry::Never)]); + let record = + KvPartnerId::from_eids(vec![eid], UnixSeconds::new(1)).expect("should build record"); + + let json = serde_json::to_string(&record).expect("should serialize"); + let back: KvPartnerId = serde_json::from_str(&json).expect("should deserialize"); + + assert_eq!( + serde_json::from_str::(&json).expect("should parse"), + serde_json::json!({ "uid": "v", "eids": [{ + "inserter": "inserter.example.com", + "matcher": "matcher.example.com", + "mm": 3, + "ext": { "k": 1 }, + "uids": [{ "id": "v", "atype": 1, "owner": "example-provider" }] + }] }), + "should store EID-level fields flat beside uids" + ); + assert_eq!(back, record, "should round-trip EID-level fields"); + } + + #[test] + fn provider_record_rejects_invalid_stored_ext() { + let big = "x".repeat(crate::ec::eid_types::MAX_EXT_BYTES); + let cases = [ + serde_json::json!({ "uid": "v", "eids": [{ "ext": [1], "uids": [{ "id": "v", "atype": 1 }] }] }), + serde_json::json!({ "uid": "v", "eids": [{ "ext": { "k": big }, "uids": [{ "id": "v", "atype": 1 }] }] }), + serde_json::json!({ "uid": "v", "eids": [{ "uids": [{ "id": "v", "atype": 1, "ext": { "k": big } }] }] }), + serde_json::json!({ "uid": "v", "eids": [{ "uids": [{ "id": "v", "atype": 1, "ext": "text" }] }] }), + ]; + + for case in cases { + let text = serde_json::to_string(&case).expect("should encode case"); + assert!( + serde_json::from_str::(&text).is_err(), + "should reject invalid stored ext: {text}" + ); + } + } + + #[test] + fn validate_rejects_oversized_inserter_and_matcher() { + let long = "a".repeat(crate::ec::eid_types::MAX_EID_FIELD_BYTES + 1); + for field in ["inserter", "matcher"] { + let mut eid = serde_json::json!({ "uids": [{ "id": "v", "atype": 1 }] }); + eid[field] = serde_json::Value::String(long.clone()); + let record: KvPartnerId = + serde_json::from_value(serde_json::json!({ "uid": "v", "eids": [eid] })) + .expect("should parse record"); + let mut entry = KvEntry::minimal("ids.example.com", "v", 1); + entry.ids.insert("ids.example.com".to_owned(), record); + + assert!(entry.validate().is_err(), "should reject oversized {field}"); + } + } + + #[test] + fn refresh_is_due_only_while_the_uid_is_live() { + let uid = KvUid::new( + UidValue::try_from("p-1").expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + None, + Expiry::At(UnixSeconds::new(5000)), + ) + .with_refresh(KvRefresh { + state: "refresh-state".to_owned(), + from: UnixSeconds::new(1000), + }); + + assert!( + !uid.refresh_due(UnixSeconds::new(999)), + "should wait for `from`" + ); + assert!( + uid.refresh_due(UnixSeconds::new(1000)), + "should be due at `from`" + ); + assert!( + !uid.refresh_due(UnixSeconds::new(5000)), + "should stop refreshing once the UID expires" + ); + } } diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index f24c489edd..b227f1dbc8 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -7,9 +7,10 @@ //! context. This is called pre-routing on every request. //! //! 2. **Generate** — [`EcContext::generate_if_needed`] creates a new EC ID -//! when none exists and consent allows it. This is called only in organic -//! handlers (publisher proxy, integration proxy) — never in read-only -//! endpoints like `/_ts/api/v1/identify`. +//! when none exists and consent allows it. This is called only for +//! publisher navigations — never in integration proxies or read-only +//! endpoints like `/_ts/api/v1/identify`. The new row is written by the +//! navigation's single page-load write in finalize ([`page_load`]). //! //! # Module structure //! @@ -26,16 +27,24 @@ //! - [`rate_limiter`] — Rate limiting abstraction (implemented by adapters) //! - [`identify`] — Identity read endpoint (`GET /_ts/api/v1/identify`) //! - [`eids`] — Shared EID resolution and formatting helpers +//! - [`eid_types`] — Value types for full-fidelity provider EIDs //! - [`batch_sync`] — S2S batch sync endpoint (`POST /_ts/api/v1/batch-sync`) //! - [`pull_sync`] — Background pull-sync dispatcher for organic routes +//! - [`page_load`] — Single-use page-load write permit and row changes +//! - [`write_plan`] — Plans a navigation's single row change +//! - [`provider`] — Identity provider interface and configuration +//! - [`coordinator`] — Server-side identity lookup coordination mod auth; pub mod admin; +pub mod after_response; pub mod batch_sync; pub mod consent; pub mod cookies; +pub mod coordinator; pub mod device; +pub mod eid_types; pub mod eids; pub mod finalize; pub mod generation; @@ -43,12 +52,15 @@ pub mod identify; pub mod kv; pub mod kv_backend; pub mod kv_types; +pub mod page_load; pub mod partner; pub mod prebid_eids; +pub mod provider; pub mod pull_sync; pub(crate) mod pull_sync_marker; pub mod rate_limiter; pub mod registry; +pub mod write_plan; /// Characters of an identifier kept when redacting it for a log. const LOG_ID_PREFIX_CHARS: usize = 8; @@ -83,8 +95,10 @@ use crate::platform::RuntimeServices; use crate::settings::Settings; use device::DeviceSignals; -use self::kv::{CreateIfAbsentOutcome, KvIdentityGraph}; +use self::coordinator::results::LookupResults; +use self::kv::KvIdentityGraph; use self::kv_types::KvEntry; +use self::page_load::{AfterResponseWrite, NavigationPermit}; use self::pull_sync_marker::{PullSyncMarkerState, validate_marker_state}; /// Bounded request classifications that may persist browser EID cookies. @@ -104,9 +118,6 @@ pub enum EidSyncSource { /// Admitted `GET /_ts/page-bids` SPA navigation. #[display("page_bids")] PageBids, - /// Request that generated a new EC identity during finalization. - #[display("new_ec")] - NewEc, } /// Request-scoped view of one EC identity-graph lookup. @@ -240,6 +251,28 @@ pub fn get_ec_id(req: &Request) -> Result, Report); + +impl Clone for PermitSlot { + fn clone(&self) -> Self { + Self(None) + } +} + +/// Holds the navigation's after-response permit. Like [`PermitSlot`], +/// cloning an [`EcContext`] never copies it. +#[derive(Debug, Default)] +struct AfterResponsePermitSlot(Option); + +impl Clone for AfterResponsePermitSlot { + fn clone(&self) -> Self { + Self(None) + } +} + /// Captures the EC state for a single request lifecycle. /// /// Created via [`read_from_request`](Self::read_from_request) during @@ -282,6 +315,15 @@ pub struct EcContext { pull_sync_marker: PullSyncMarkerState, /// Allowed returning-user EID persistence source, assigned only after request filters pass. eid_sync_source: Option, + /// Navigation permit issued by the router for page-load navigations. + navigation_permit: PermitSlot, + /// Permit for the navigation's follow-up write after its response, set + /// by finalize when it confirms the navigation permit. + after_response_permit: AfterResponsePermitSlot, + /// Validated provider lookup results collected for this navigation. + lookup_results: LookupResults, + /// New row built by [`Self::generate_if_needed`], written by finalize. + pending_entry: Option>, } impl EcContext { @@ -364,14 +406,21 @@ impl EcContext { client_eids: None, pull_sync_marker: PullSyncMarkerState::from_cookie(parsed.pull_sync_marker), eid_sync_source: None, + navigation_permit: PermitSlot::default(), + after_response_permit: AfterResponsePermitSlot::default(), + lookup_results: LookupResults::default(), + pending_entry: None, }) } /// Generates a new EC ID if none exists and consent allows it. /// - /// This is the second phase of the EC lifecycle. Call this only in - /// organic handlers (publisher proxy, integration proxy, auction) — - /// never in read-only endpoints. + /// This is the second phase of the EC lifecycle. Call this only for + /// publisher navigations — never in read-only endpoints. + /// + /// Never writes KV. When `kv` is available, the new row is held as the + /// pending entry and created by the navigation's single page-load write + /// in finalize; the EC is issued only if that write succeeds. /// /// If an EC ID already exists (from the request), this is a no-op. /// If consent does not permit EC creation, this is a no-op. @@ -403,62 +452,32 @@ impl EcContext { }) })?; - const MAX_CREATE_ATTEMPTS: usize = 5; - for attempt in 0..MAX_CREATE_ATTEMPTS { - let ec_id = generation::generate_ec_id(settings, client_ip)?; - log::info!("Generated new EC ID: {}", log_id(&ec_id)); - let now = current_timestamp(); - let mut entry = KvEntry::new( - &self.consent, - self.geo_info.as_ref(), - now, - &settings.publisher.domain, - ); - entry.device = self - .device_signals - .as_ref() - .map(DeviceSignals::to_kv_device); - - if let Some(graph) = kv { - match graph.create_if_absent(&ec_id, &entry) { - Ok(CreateIfAbsentOutcome::Written) => { - self.kv_snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(entry), - generation: None, - }; - } - Ok(CreateIfAbsentOutcome::AlreadyExists) => { - log::warn!( - "Generated EC ID collision on attempt {}/{MAX_CREATE_ATTEMPTS}", - attempt + 1 - ); - continue; - } - Err(err) => { - log::error!( - "Failed to create EC entry for id '{}' after generation: {err:?}", - log_id(&ec_id), - ); - return Err(err.change_context(TrustedServerError::EdgeCookie { - message: "Failed to persist generated EC ID to KV identity graph" - .to_string(), - })); - } - } - } - - self.ec_value = Some(ec_id); - self.ec_generated = true; - self.pull_sync_marker.invalidate_for_replaced_ec(); - return Ok(()); + let ec_id = generation::generate_ec_id(settings, client_ip)?; + log::info!("Generated new EC ID: {}", log_id(&ec_id)); + let now = current_timestamp(); + let mut entry = KvEntry::new( + &self.consent, + self.geo_info.as_ref(), + now, + &settings.publisher.domain, + ); + entry.device = self + .device_signals + .as_ref() + .map(DeviceSignals::to_kv_device); + + if kv.is_some() { + // The row is created by the navigation's single page-load write in + // finalize; a collision there leaves this EC unissued. + self.pending_entry = Some(Box::new(entry)); + self.kv_snapshot = EcKvSnapshot::Missing { + ec_id: ec_id.clone(), + }; } - - Err(Report::new(TrustedServerError::EdgeCookie { - message: format!( - "Failed to allocate a unique EC ID after {MAX_CREATE_ATTEMPTS} attempts" - ), - })) + self.ec_value = Some(ec_id); + self.ec_generated = true; + self.pull_sync_marker.invalidate_for_replaced_ec(); + Ok(()) } /// Returns the EC ID value, if present (either from request or generated). @@ -607,6 +626,53 @@ impl EcContext { self.pull_sync_marker = state; } + /// Stores the navigation permit issued by the router. + pub fn set_navigation_permit(&mut self, permit: NavigationPermit) { + self.navigation_permit = PermitSlot(Some(permit)); + } + + /// Whether this request still holds its page-load write permit. + pub(crate) fn has_navigation_permit(&self) -> bool { + self.navigation_permit.0.is_some() + } + + /// Takes the navigation permit; returns `None` after the first call. + pub(crate) fn take_navigation_permit(&mut self) -> Option { + self.navigation_permit.0.take() + } + + /// Stores the after-response permit confirmed with the navigation permit. + pub(crate) fn set_after_response_permit(&mut self, permit: AfterResponseWrite) { + self.after_response_permit = AfterResponsePermitSlot(Some(permit)); + } + + /// Takes the after-response permit; returns `None` when finalize did not + /// confirm a navigation, or after the first call. + pub fn take_after_response_permit(&mut self) -> Option { + self.after_response_permit.0.take() + } + + /// Takes the navigation permit, for adapter tests. + #[cfg(any(test, feature = "test-utils"))] + pub fn take_navigation_permit_for_test(&mut self) -> Option { + self.take_navigation_permit() + } + + /// Stores validated provider lookup results for finalize. + pub fn set_lookup_results(&mut self, results: LookupResults) { + self.lookup_results = results; + } + + /// Provider lookup results collected for this navigation. + pub(crate) fn lookup_results(&self) -> &LookupResults { + &self.lookup_results + } + + /// Takes the row built by [`Self::generate_if_needed`]. + pub(crate) fn take_pending_entry(&mut self) -> Option { + self.pending_entry.take().map(|entry| *entry) + } + /// Replaces an orphaned active ID after its new backing row is persisted. pub(crate) fn replace_with_generated(&mut self, ec_id: String, snapshot: EcKvSnapshot) { self.ec_value = Some(ec_id); @@ -665,6 +731,10 @@ impl EcContext { client_eids: None, pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, + navigation_permit: PermitSlot::default(), + after_response_permit: AfterResponsePermitSlot::default(), + lookup_results: LookupResults::default(), + pending_entry: None, } } @@ -690,6 +760,10 @@ impl EcContext { client_eids: None, pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, + navigation_permit: PermitSlot::default(), + after_response_permit: AfterResponsePermitSlot::default(), + lookup_results: LookupResults::default(), + pending_entry: None, } } @@ -718,6 +792,10 @@ impl EcContext { client_eids: None, pull_sync_marker: PullSyncMarkerState::Absent, eid_sync_source: None, + navigation_permit: PermitSlot::default(), + after_response_permit: AfterResponsePermitSlot::default(), + lookup_results: LookupResults::default(), + pending_entry: None, } } } @@ -728,7 +806,8 @@ impl EcContext { /// native and `wasm32-wasip1` targets and to a JS-backed clock on /// `wasm32-unknown-unknown` (Cloudflare Workers), where `std::time` is not /// available. -pub(crate) fn current_timestamp() -> u64 { +#[must_use] +pub fn current_timestamp() -> u64 { checked_current_timestamp().unwrap_or(0) } @@ -766,69 +845,9 @@ mod tests { use super::*; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ConsentContext, ConsentSource}; - use crate::ec::kv_backend::test_support::InMemoryEcKv; - use crate::ec::kv_backend::{ - EcKvLookup, EcKvStore, EcKvWrite, EcKvWriteMode, EcKvWriteOutcome, - }; use crate::platform::test_support::noop_services; use crate::test_support::tests::create_test_settings; - /// [`EcKvStore`] wrapper whose first `collisions` `Add` writes report a - /// precondition failure, forcing generation to retry with a fresh suffix. - struct AddCollidingEcKv { - inner: InMemoryEcKv, - collisions_remaining: std::sync::Mutex, - } - - impl AddCollidingEcKv { - fn new(collisions: u32) -> Self { - Self { - inner: InMemoryEcKv::new("add-colliding-store"), - collisions_remaining: std::sync::Mutex::new(collisions), - } - } - } - - impl EcKvStore for AddCollidingEcKv { - fn store_name(&self) -> &str { - self.inner.store_name() - } - fn lookup(&self, key: &str) -> Result, Report> { - self.inner.lookup(key) - } - fn key_exists(&self, key: &str) -> Result> { - self.inner.key_exists(key) - } - - fn insert( - &self, - key: &str, - write: EcKvWrite<'_>, - ) -> Result> { - if matches!(write.mode, EcKvWriteMode::Add) { - let mut remaining = self - .collisions_remaining - .lock() - .expect("should lock collision counter"); - if *remaining > 0 { - *remaining -= 1; - return Ok(EcKvWriteOutcome::PreconditionFailed); - } - } - self.inner.insert(key, write) - } - fn list_keys_with_prefix( - &self, - prefix: &str, - limit: u32, - ) -> Result, Report> { - self.inner.list_keys_with_prefix(prefix, limit) - } - fn delete(&self, key: &str) -> Result<(), Report> { - self.inner.delete(key) - } - } - fn granting_consent() -> ConsentContext { ConsentContext { jurisdiction: Jurisdiction::NonRegulated, @@ -838,37 +857,48 @@ mod tests { } #[test] - fn generate_if_needed_retries_id_collision_then_persists() { + fn generate_if_needed_defers_row_creation_to_the_page_load_write() { let settings = create_test_settings(); - let mut ec = - EcContext::new_for_test_with_ip(None, granting_consent(), Some("192.0.2.5".to_owned())); - let graph = KvIdentityGraph::new(AddCollidingEcKv::new(2)); + let mut ec_context = EcContext::new_for_test_with_ip( + None, + granting_consent(), + Some("192.0.2.10".to_owned()), + ); + let graph = KvIdentityGraph::in_memory("gen"); - ec.generate_if_needed(&settings, Some(&graph)) - .expect("should generate after bounded collisions"); + ec_context + .generate_if_needed(&settings, Some(&graph)) + .expect("should mint EC"); - assert!(ec.ec_value().is_some(), "should allocate a fresh EC ID"); - assert!(ec.ec_generated(), "should mark the EC as generated"); + let ec_id = ec_context.ec_value().expect("should set EC").to_owned(); assert!( - matches!(ec.kv_snapshot(), EcKvSnapshot::Present { .. }), - "generation should seed a present snapshot" + graph.get(&ec_id).expect("should read").is_none(), + "should not write during generation" + ); + assert!( + ec_context.take_pending_entry().is_some(), + "should hold the new row for finalize" + ); + assert!( + matches!(ec_context.kv_snapshot(), EcKvSnapshot::Missing { .. }), + "should record missing row" ); } #[test] - fn generate_if_needed_errors_after_collision_exhaustion() { - let settings = create_test_settings(); - let mut ec = - EcContext::new_for_test_with_ip(None, granting_consent(), Some("192.0.2.6".to_owned())); - // Collide on every attempt so the bounded retry is exhausted. - let graph = KvIdentityGraph::new(AddCollidingEcKv::new(u32::MAX)); + fn cloned_context_never_carries_the_navigation_permit() { + let mut ec_context = EcContext::default(); + ec_context.set_navigation_permit(NavigationPermit::for_test()); - let result = ec.generate_if_needed(&settings, Some(&graph)); + let mut clone = ec_context.clone(); - assert!(result.is_err(), "should fail after exhausting attempts"); assert!( - ec.ec_value().is_none() && !ec.ec_generated(), - "must not activate an EC ID it could not persist" + clone.take_navigation_permit().is_none(), + "should drop permit on clone" + ); + assert!( + ec_context.take_navigation_permit().is_some(), + "should keep permit on original" ); } diff --git a/crates/trusted-server-core/src/ec/page_load.rs b/crates/trusted-server-core/src/ec/page_load.rs new file mode 100644 index 0000000000..38125a0f10 --- /dev/null +++ b/crates/trusted-server-core/src/ec/page_load.rs @@ -0,0 +1,367 @@ +//! The one-KV-write-per-page-load rule, enforced by types. +//! +//! A request classified as a top-level document navigation receives a +//! [`NavigationPermit`]. Once the final response status is known it becomes a +//! [`PageLoadWrite`], which [`KvIdentityGraph::commit_page_load`] consumes. +//! Neither type is `Clone`, so one navigation makes at most one write before +//! its response. +//! +//! Two writes fall outside this rule. [`AfterResponseWrite`], confirmed +//! alongside the page load's permit, allows one follow-up update after the +//! response has been sent, carrying only the results of after-response +//! identity lookups. A consent withdrawal's tombstone is a privacy write on +//! any request that carries the EC cookie, and takes no permit. +//! +//! [`KvIdentityGraph::commit_page_load`]: super::kv::KvIdentityGraph::commit_page_load + +use edgezero_core::body::Body as EdgeBody; +use http::{Method, Request, StatusCode}; + +use super::EcKvSnapshot; +use super::kv_types::KvEntry; + +/// Issued to a request classified as a page-load navigation. +#[derive(Debug)] +pub struct NavigationPermit(()); + +impl NavigationPermit { + /// Returns a permit when `request` is a top-level document navigation: + /// `GET`, `Sec-Fetch-Dest: document`, `Sec-Fetch-Mode: navigate`, and no + /// `prefetch` token in `Sec-Purpose`. + /// + /// Call once per request, from the publisher-proxy route only. + #[must_use] + pub fn classify(request: &Request) -> Option { + let headers = request.headers(); + let header_is = |name: &str, expected: &str| { + headers + .get(name) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.trim().eq_ignore_ascii_case(expected)) + }; + let prefetch = headers + .get("sec-purpose") + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| { + value + .split([';', ',']) + .any(|token| token.trim().eq_ignore_ascii_case("prefetch")) + }); + let navigation = request.method() == Method::GET + && header_is("sec-fetch-dest", "document") + && header_is("sec-fetch-mode", "navigate") + && !prefetch; + navigation.then_some(Self(())) + } + + /// Confirms the permit once the final response status is known. + /// + /// Returns `None` for a redirect (301, 302, 303, 307, 308): the redirect + /// target is the page load. Other 3xx statuses, such as 304 Not Modified, + /// are the page load itself and keep the permit. + #[must_use] + pub fn confirm(self, status: StatusCode) -> Option { + (!Self::is_redirect(status)).then_some(PageLoadWrite(())) + } + + /// Confirms the permit like [`Self::confirm`] and also issues the + /// navigation's [`AfterResponseWrite`]. + /// + /// Returns `None` for a redirect, which makes neither write. + #[must_use] + pub fn confirm_with_after_response( + self, + status: StatusCode, + ) -> Option<(PageLoadWrite, AfterResponseWrite)> { + self.confirm(status) + .map(|permit| (permit, AfterResponseWrite(()))) + } + + /// Whether `status` is a redirect (301, 302, 303, 307, 308) on which the + /// page load's write is skipped. + /// + /// Callers use this to drop work, such as pending identity lookups, whose + /// only purpose is that write. + #[must_use] + pub fn is_redirect(status: StatusCode) -> bool { + matches!( + status, + StatusCode::MOVED_PERMANENTLY + | StatusCode::FOUND + | StatusCode::SEE_OTHER + | StatusCode::TEMPORARY_REDIRECT + | StatusCode::PERMANENT_REDIRECT + ) + } + + /// Builds a permit without a request, for tests. + #[cfg(test)] + #[must_use] + pub fn for_test() -> Self { + Self(()) + } +} + +/// Proof that the current request may make the page load's single KV write. +#[derive(Debug)] +pub struct PageLoadWrite(()); + +impl PageLoadWrite { + /// Consumes the permit. Called only by the write itself. + pub(crate) fn spend(self) {} +} + +/// Proof that the current navigation may make one follow-up update after its +/// response has been sent. +/// +/// Issued only by [`NavigationPermit::confirm_with_after_response`] and +/// consumed by [`KvIdentityGraph::commit_after_response`]. It is not `Clone`, +/// so a navigation makes at most one follow-up write. +/// +/// [`KvIdentityGraph::commit_after_response`]: super::kv::KvIdentityGraph::commit_after_response +#[derive(Debug)] +pub struct AfterResponseWrite(()); + +impl AfterResponseWrite { + /// Consumes the permit. Called only by the write itself. + pub(crate) fn spend(self) {} + + /// Builds a permit without a navigation, for tests. + #[cfg(test)] + #[must_use] + pub fn for_test() -> Self { + Self(()) + } +} + +/// KV generation a conditional write expects. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Generation(u64); + +impl Generation { + /// Reads the generation captured in `snapshot` for `ec_id`. + /// + /// Returns `None` when the snapshot has no generation, so no blind + /// overwrite can be planned. + #[must_use] + pub fn of(snapshot: &EcKvSnapshot, ec_id: &str) -> Option { + snapshot.generation_for(ec_id).map(Self) + } + + /// The raw generation. + #[must_use] + pub const fn get(self) -> u64 { + self.0 + } +} + +/// The single row change of a navigation, with its CAS precondition. +#[derive(Debug)] +pub enum RowChange { + /// Create a row that must not exist (new EC or orphan recovery). + Create(KvEntry), + /// Replace the row only if it is still at `expected`. + Update { + /// The full new row. + entry: KvEntry, + /// Generation the row must still have. + expected: Generation, + }, +} + +/// Why the single write did not land. +#[derive(Debug, derive_more::Display)] +pub enum KvWriteError { + /// The row changed since it was read. + #[display("row changed since it was read")] + Conflict, + /// A create found an existing row. + #[display("row already exists")] + AlreadyExists, + /// The KV store rejected the write. + #[display("KV store rejected the write")] + Store, +} + +impl core::error::Error for KvWriteError {} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ec::kv::KvIdentityGraph; + use crate::ec::kv_types::KvEntry; + + fn request(headers: &[(&str, &str)]) -> http::Request { + let mut builder = http::Request::builder() + .method(http::Method::GET) + .uri("https://www.example.com/"); + for (name, value) in headers { + builder = builder.header(*name, *value); + } + builder + .body(EdgeBody::empty()) + .expect("should build request") + } + + const DOCUMENT: [(&str, &str); 2] = [ + ("sec-fetch-dest", "document"), + ("sec-fetch-mode", "navigate"), + ]; + + #[test] + fn classify_accepts_top_level_document_navigation() { + assert!( + NavigationPermit::classify(&request(&DOCUMENT)).is_some(), + "should permit navigation" + ); + } + + #[test] + fn classify_rejects_missing_fetch_metadata_iframes_and_prefetch() { + assert!( + NavigationPermit::classify(&request(&[])).is_none(), + "should reject without fetch metadata" + ); + assert!( + NavigationPermit::classify(&request(&[ + ("sec-fetch-dest", "iframe"), + ("sec-fetch-mode", "navigate") + ])) + .is_none(), + "should reject iframe" + ); + let mut prefetch = DOCUMENT.to_vec(); + prefetch.push(("sec-purpose", "prefetch;prerender")); + assert!( + NavigationPermit::classify(&request(&prefetch)).is_none(), + "should reject prefetch" + ); + } + + #[test] + fn confirm_rejects_redirects() { + assert!( + NavigationPermit::for_test() + .confirm(http::StatusCode::FOUND) + .is_none(), + "should not write on redirect" + ); + assert!( + NavigationPermit::for_test() + .confirm(http::StatusCode::NOT_FOUND) + .is_some(), + "should write on error page" + ); + } + + #[test] + fn confirm_keeps_not_modified_and_drops_redirects() { + assert!( + NavigationPermit::for_test() + .confirm(http::StatusCode::NOT_MODIFIED) + .is_some(), + "should write on 304 Not Modified" + ); + for status in [ + http::StatusCode::FOUND, + http::StatusCode::TEMPORARY_REDIRECT, + ] { + assert!( + NavigationPermit::for_test().confirm(status).is_none(), + "should not write on redirect {status}" + ); + } + } + + #[test] + fn is_redirect_matches_only_write_skipping_statuses() { + for status in [ + http::StatusCode::MOVED_PERMANENTLY, + http::StatusCode::FOUND, + http::StatusCode::SEE_OTHER, + http::StatusCode::TEMPORARY_REDIRECT, + http::StatusCode::PERMANENT_REDIRECT, + ] { + assert!( + NavigationPermit::is_redirect(status), + "should treat {status} as a redirect" + ); + } + for status in [http::StatusCode::OK, http::StatusCode::NOT_MODIFIED] { + assert!( + !NavigationPermit::is_redirect(status), + "should not treat {status} as a redirect" + ); + } + } + + fn permit() -> PageLoadWrite { + NavigationPermit::for_test() + .confirm(http::StatusCode::OK) + .expect("should confirm") + } + + fn live_entry() -> KvEntry { + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry + } + + #[test] + fn commit_create_reports_existing_row() { + let graph = KvIdentityGraph::in_memory("page_load"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + graph + .commit_page_load(permit(), &ec_id, RowChange::Create(live_entry())) + .expect("should create row"); + + let error = graph + .commit_page_load(permit(), &ec_id, RowChange::Create(live_entry())) + .expect_err("should refuse second create"); + + assert!( + matches!(error.current_context(), KvWriteError::AlreadyExists), + "should report existing row" + ); + } + + #[test] + fn commit_update_with_stale_generation_conflicts_without_retry() { + let graph = KvIdentityGraph::in_memory("page_load"); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + graph + .create(&ec_id, &live_entry()) + .expect("should seed row"); + let snapshot = graph.load_snapshot(&ec_id); + let expected = Generation::of(&snapshot, &ec_id).expect("should read generation"); + graph + .create_or_revive(&ec_id, &live_entry()) + .expect("should no-op on live row"); + graph + .commit_page_load( + permit(), + &ec_id, + RowChange::Update { + entry: live_entry(), + expected, + }, + ) + .expect("should write at current generation"); + + let error = graph + .commit_page_load( + permit(), + &ec_id, + RowChange::Update { + entry: live_entry(), + expected, + }, + ) + .expect_err("should reject stale generation"); + + assert!( + matches!(error.current_context(), KvWriteError::Conflict), + "should report conflict" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/prebid_eids.rs b/crates/trusted-server-core/src/ec/prebid_eids.rs index d823cb32fb..8619fd0651 100644 --- a/crates/trusted-server-core/src/ec/prebid_eids.rs +++ b/crates/trusted-server-core/src/ec/prebid_eids.rs @@ -157,6 +157,7 @@ pub(crate) fn collect_eid_updates( { updates.push(update); } + // Provider-owned sources stay in the list; the write planner skips them. dedupe_partner_updates(updates) } @@ -294,17 +295,15 @@ fn structured_cookie_eids_to_openrtb(entries: Vec) -> Vec Option { - if uid.id.is_empty() { + // Cookie UIDs keep the browser limit; only provider writes may be longer. + if uid.id.is_empty() || uid.id.len() > MAX_UID_LENGTH { return None; } @@ -324,14 +323,18 @@ fn structured_cookie_uid_to_openrtb(uid: StructuredCookieUid) -> Option { fn legacy_cookie_eids_to_openrtb(entries: Vec) -> Vec { entries .into_iter() - .filter(|entry| !entry.source.is_empty() && !entry.id.is_empty()) - .map(|entry| Eid { - source: entry.source, - uids: vec![Uid { - id: entry.id, - atype: (entry.atype >= 0).then_some(entry.atype), - ext: None, - }], + .filter(|entry| { + !entry.source.is_empty() && !entry.id.is_empty() && entry.id.len() <= MAX_UID_LENGTH + }) + .map(|entry| { + Eid::new( + entry.source, + vec![Uid { + id: entry.id, + atype: (entry.atype >= 0).then_some(entry.atype), + ext: None, + }], + ) }) .collect() } @@ -450,6 +453,35 @@ mod tests { ); } + #[test] + fn parse_prebid_eids_cookie_drops_uids_over_the_browser_limit() { + // Arrange + let long = "x".repeat(MAX_UID_LENGTH + 1); + let at_limit = "y".repeat(MAX_UID_LENGTH); + let structured = BASE64.encode( + json!([{ "source": "ids.example.com", "uids": [ + { "id": long, "atype": 1 }, + { "id": at_limit, "atype": 1 } + ]}]) + .to_string(), + ); + let legacy = BASE64 + .encode(json!([{ "source": "ids.example.com", "id": long, "atype": 1 }]).to_string()); + + // Act + let structured = parse_prebid_eids_cookie(&structured).expect("should decode"); + let legacy = parse_prebid_eids_cookie(&legacy).expect("should decode"); + + // Assert + let ids: Vec = structured[0].uids.iter().map(|uid| uid.id.len()).collect(); + assert_eq!( + ids, + vec![MAX_UID_LENGTH], + "should keep only the UID within the limit" + ); + assert!(legacy.is_empty(), "should drop an over-limit legacy UID"); + } + #[test] fn parse_prebid_eids_cookie_rejects_invalid_base64() { let result = parse_prebid_eids_cookie("not-valid-base64!!!"); @@ -636,22 +668,22 @@ mod tests { {"source": "id5-sync.com", "uids": [{"id": "ID5_stale", "atype": 1}]} ])); let body_eids = vec![ - Eid { - source: "id5-sync.com".to_owned(), - uids: vec![Uid { + Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { id: "ID5_fresh".to_owned(), atype: Some(1), ext: None, }], - }, - Eid { - source: "liveramp.com".to_owned(), - uids: vec![Uid { + ), + Eid::new( + "liveramp.com".to_owned(), + vec![Uid { id: "LR_xyz".to_owned(), atype: Some(3), ext: None, }], - }, + ), ]; let updates = collect_eid_updates(Some(&eids_cookie), None, Some(&body_eids), ®istry); diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs new file mode 100644 index 0000000000..9f479fa296 --- /dev/null +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -0,0 +1,926 @@ +//! Identity provider interface (spec §7). +//! +//! Providers are pure: they build a lookup request and parse its response. +//! They never perform HTTP, KV, cookie or clock work; the coordinator in +//! [`crate::ec::coordinator`] executes requests and applies consent, +//! transport, validation, expiry and persistence policy. +//! +//! Provider implementations live in the `trusted-server-identity-providers` +//! crate, which depends on core; core never depends on it. + +use core::fmt; +use std::borrow::Cow; +use std::collections::HashSet; +use std::net::IpAddr; +use std::num::NonZeroUsize; +use std::time::Duration; + +use error_stack::{Report, ResultExt as _}; +use serde::Deserialize as _; +use serde::de::DeserializeOwned; +use serde_json::value::RawValue; +use url::Url; + +use crate::consent::ConsentContext; +use crate::ec::eid_types::{ProviderId, SourceDomain, UnixSeconds}; +use crate::ec::kv_types::{KvEid, KvEntry}; +use crate::ec::registry::PartnerRegistry; +use crate::error::TrustedServerError; +use crate::settings::{Ec, IdentityProviderConfig, ProviderSecrets}; + +/// Builds lookup requests and parses lookup responses for one identity provider. +/// +/// Implementations must not perform I/O or read the clock; the coordinator +/// executes requests and applies consent, transport, validation, expiry and +/// persistence policy. Coordinators are built once and shared for the +/// application's lifetime, so implementations are `Send + Sync`. +pub trait IdentityProvider: Send + Sync { + /// Describes which sources and EID shapes this provider can return. + fn capabilities(&self) -> &ProviderCapabilities; + + /// Builds the lookup request for the given due sources. + /// + /// # Errors + /// + /// Returns [`ProviderError::Request`] when the input cannot form a request. + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result>; + + /// Parses a provider response, borrowing from its body. + /// + /// # Errors + /// + /// Returns [`ProviderError::InvalidResponse`] when the body cannot be + /// parsed at all. Individually malformed entries are returned and dropped + /// by the coordinator's validation, not reported as an error. + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report>; + + /// Creates IDs for due sources without a lookup, for a provider whose + /// IDs the operator owns rather than a vendor server. + /// + /// Runs when a source is due. A returned entry goes through the same + /// validation and write plan as a lookup result, and no request is built. + /// The default creates nothing. + fn generate(&self, lookup: &LookupInput<'_>) -> Option>> { + let _ = lookup; + None + } +} + +impl IdentityProvider for Box

{ + fn capabilities(&self) -> &ProviderCapabilities { + (**self).capabilities() + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + (**self).build_request(lookup) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + (**self).parse_response(response) + } + + fn generate(&self, lookup: &LookupInput<'_>) -> Option>> { + (**self).generate(lookup) + } +} + +/// What a provider can return, used by the coordinator to apply shared policy. +#[derive(Debug, Clone)] +pub struct ProviderCapabilities { + /// Sources this implementation can return and the input each needs. + pub sources: Box<[SourceCapability]>, + /// Limits and fields accepted from one response. + pub shape: EidShape, +} + +/// One source a provider can return. +#[derive(Debug, Clone)] +pub struct SourceCapability { + /// The source domain. + pub source: SourceDomain, + /// Visitor input the source needs. + pub input: SourceInput, +} + +/// Visitor input a source needs before it can be looked up. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceInput { + /// Works for any visitor. + Anonymous, + /// Needs a hashed email. Always skipped in V1. + HashedEmail, +} + +/// Limits a provider declares for one response. +#[derive(Debug, Clone, Copy)] +pub struct EidShape { + /// Maximum EID entries kept from one response. + pub max_eids: NonZeroUsize, + /// Maximum UIDs kept per EID entry. + pub max_uids_per_eid: NonZeroUsize, + /// Whether EID-level fields are kept. + pub eid_level_fields: EidLevelFields, +} + +/// Whether `inserter`, `matcher`, `mm` and EID `ext` are kept. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EidLevelFields { + /// The fields are validated and stored. + Preserved, + /// The fields are dropped without rejecting the entry. + Ignored, +} + +/// Request-scoped data the coordinator lends to a provider. +#[derive(Debug, Clone, Copy)] +pub struct LookupInput<'a> { + /// Active EC ID. + pub ec_id: &'a str, + /// Request consent. + pub consent: &'a ConsentContext, + /// Client signals. + pub client: ClientSignals<'a>, + /// Publisher domain. + pub publisher_domain: &'a str, + /// Sources due for this provider, already filtered by consent and input. + pub due_sources: &'a [SourceDomain], + /// This provider's currently stored entries, borrowed from the row. + pub current: OwnedEntries<'a>, + /// Always `None` in V1. + pub hashed_email: Option<&'a HashedEmail>, +} + +/// Client signals lent to providers. +#[derive(Debug, Clone, Copy)] +pub struct ClientSignals<'a> { + /// Client IP address. + pub ip: Option, + /// Client `User-Agent`. + pub user_agent: Option<&'a str>, +} + +/// This provider's stored entries, borrowed from the row. +/// +/// Only entries whose every UID this provider wrote are visible, so another +/// provider's refresh state never reaches it. +#[derive(Debug, Clone, Copy)] +pub struct OwnedEntries<'a> { + entry: Option<&'a KvEntry>, + sources: &'a [SourceDomain], + owner: Option<&'a ProviderId>, +} + +impl<'a> OwnedEntries<'a> { + /// Views the entries of `entry` for `sources` that `owner` wrote. + #[must_use] + pub fn new( + entry: Option<&'a KvEntry>, + sources: &'a [SourceDomain], + owner: &'a ProviderId, + ) -> Self { + Self { + entry, + sources, + owner: Some(owner), + } + } + + /// A view holding no entries, for a provider with nothing stored. + #[must_use] + pub fn empty() -> Self { + Self { + entry: None, + sources: &[], + owner: None, + } + } + + /// Iterates `(source, entry)` pairs in source order. + pub fn iter(&self) -> impl Iterator + 'a { + let entry = self.entry; + let owner = self.owner; + self.sources.iter().flat_map(move |source| { + entry + .and_then(|entry| entry.ids.get_key_value(source.as_str())) + .into_iter() + .flat_map(move |(key, record)| { + record + .eids() + .iter() + .filter(move |eid| eid.uids().iter().all(|uid| uid.owner() == owner)) + .map(move |eid| (key.as_str(), eid)) + }) + }) + } +} + +/// A validated lowercase hex SHA-256 email digest. V1 never constructs one. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HashedEmail(Box); + +impl HashedEmail { + /// Returns the digest. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +/// An outbound request description. The coordinator, not the provider, sends it. +/// +/// `Debug` is implemented by hand and prints only the method and host, because +/// headers and query strings may carry secrets. Providers mark secret headers +/// with `HeaderValue::set_sensitive(true)`. +pub struct ProviderRequest { + /// HTTP method. + pub method: http::Method, + /// Request URL; must be `https` with an allowlisted host. + pub url: Url, + /// Request headers. + pub headers: http::HeaderMap, + /// Request body. + pub body: bytes::Bytes, + /// Provider state returned with the response in + /// [`ProviderResponse::context`]; never sent. + pub context: Option, +} + +impl fmt::Debug for ProviderRequest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ProviderRequest") + .field("method", &self.method) + .field("host", &self.url.host_str()) + .finish_non_exhaustive() + } +} + +/// A provider response, borrowed from the transport. +/// +/// Its `Debug` output omits the body, headers and context, which can hold +/// identifiers or a reply-decryption key. +#[derive(Clone, Copy)] +pub struct ProviderResponse<'body> { + /// Response status. + pub status: http::StatusCode, + /// Response headers. + pub headers: &'body http::HeaderMap, + /// Response body. + pub body: &'body [u8], + /// The [`ProviderRequest::context`] of the request this answers. + pub context: Option<&'body str>, +} + +impl fmt::Debug for ProviderResponse<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ProviderResponse") + .field("status", &self.status) + .field("body_len", &self.body.len()) + .field("has_context", &self.context.is_some()) + .finish_non_exhaustive() + } +} + +/// A parsed provider response. +#[derive(Debug)] +pub enum LookupOutcome<'body> { + /// Entries were returned. + Found(Vec>), + /// The provider knows no ID for this visitor. + NoMatch, + /// The visitor opted out with the provider; its stored UIDs are removed. + OptedOut, +} + +/// One entry as returned by a provider, before validation. +#[derive(Debug)] +pub struct ProviderEid<'body> { + /// EID source domain. + pub source: Cow<'body, str>, + /// `OpenRTB` `inserter`. + pub inserter: Option>, + /// `OpenRTB` `matcher`. + pub matcher: Option>, + /// `OpenRTB` `mm`. + pub mm: Option, + /// EID-level `ext`. + pub ext: Option<&'body RawValue>, + /// UIDs of the entry. + pub uids: Vec>, +} + +/// One UID as returned by a provider, before validation. +#[derive(Debug)] +pub struct ProviderUid<'body> { + /// UID value. + pub id: Cow<'body, str>, + /// Agent type; filled from partner config when missing. + pub atype: Option, + /// UID-level `ext`. + pub ext: Option<&'body RawValue>, + /// Expiry as the provider states it. + pub expiry: ProviderExpiry, + /// Private refresh state kept with the UID in the row and never served. + pub refresh: Option>, +} + +/// Refresh state a provider asks the framework to keep with a UID. +/// +/// The stored UID's source becomes due again at `from`, and the state is +/// handed back through [`LookupInput::current`]. `Debug` hides the state, +/// which may hold a credential. +pub struct ProviderRefresh<'body> { + /// Provider-defined state, such as a refresh token. + pub state: Cow<'body, str>, + /// When the provider should be asked to refresh the UID. + pub from: ProviderExpiry, +} + +impl fmt::Debug for ProviderRefresh<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ProviderRefresh") + .field("from", &self.from) + .finish_non_exhaustive() + } +} + +/// Expiry as a provider states it. The coordinator turns it into +/// [`crate::ec::eid_types::Expiry`] using its own clock, so providers never +/// read time. +#[derive(Debug, Clone, Copy)] +pub enum ProviderExpiry { + /// No expiry was stated. + Unspecified, + /// Expires at this time. + At(UnixSeconds), + /// Expires this long after the request. + After(Duration), +} + +/// Why a provider could not build a request or parse a response. +#[derive(Debug, derive_more::Display)] +pub enum ProviderError { + /// The input cannot form a request. + #[display("provider request could not be built")] + Request, + /// The body cannot be parsed at all. + #[display("provider response could not be parsed")] + InvalidResponse, +} + +impl core::error::Error for ProviderError {} + +/// Typed constructor for one provider implementation. +pub trait ProviderKind { + /// Value of `kind` in `[[ec.identity_providers]]`. + const KIND: &'static str; + /// The provider-specific `settings` table. + type Settings: DeserializeOwned; + /// The constructed provider. + type Provider: IdentityProvider + 'static; + + /// Builds the provider from shared and provider-specific configuration. + /// + /// # Errors + /// + /// Returns [`ProviderConfigError::Invalid`] when the settings are unusable. + fn build( + common: &ProviderCommon<'_>, + settings: Self::Settings, + ) -> Result>; +} + +/// Shared configuration every provider receives at build time. +#[derive(Debug, Clone, Copy)] +pub struct ProviderCommon<'a> { + /// Configured lookup endpoint. + pub endpoint: &'a Url, + /// Configured sources. + pub sources: &'a [SourceDomain], + /// Resolved secrets. + pub secrets: &'a ProviderSecrets, +} + +/// Why a provider configuration was rejected. +#[derive(Debug, derive_more::Display)] +pub enum ProviderConfigError { + /// No implementation is registered for the configured `kind`. + #[display("provider kind is not registered")] + UnknownKind, + /// The configuration is unusable. + #[display("provider configuration is invalid")] + Invalid, +} + +impl core::error::Error for ProviderConfigError {} + +/// Parses `config.settings` for `K` and builds a boxed provider. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError::Invalid`] when the settings do not +/// deserialize into `K::Settings`, a source or the endpoint is invalid, or +/// [`ProviderKind::build`] rejects them. +pub fn build_kind( + config: &IdentityProviderConfig, + secrets: &ProviderSecrets, +) -> Result, Report> { + let settings = K::Settings::deserialize(serde_json::Value::Object(config.settings.clone())) + .change_context(ProviderConfigError::Invalid) + .attach_with(|| format!("settings for provider `{}` are invalid", config.id))?; + let endpoint = Url::parse(&config.endpoint) + .change_context(ProviderConfigError::Invalid) + .attach_with(|| format!("endpoint for provider `{}` is not a URL", config.id))?; + let sources = parse_sources(config)?; + let common = ProviderCommon { + endpoint: &endpoint, + sources: &sources, + secrets, + }; + let provider = K::build(&common, settings)?; + Ok(Box::new(provider)) +} + +/// Parses and normalizes a provider's configured sources. +pub(crate) fn parse_sources( + config: &IdentityProviderConfig, +) -> Result, Report> { + config + .sources + .iter() + .map(|source| SourceDomain::try_from(source.as_str())) + .collect::, _>>() + .change_context(ProviderConfigError::Invalid) + .attach_with(|| format!("provider `{}` has an invalid source", config.id)) +} + +/// Whether `url` is `https` and its host is one of `allowed_domains`, +/// compared ASCII case-insensitively. +#[must_use] +pub(crate) fn destination_allowed>(url: &Url, allowed_domains: &[S]) -> bool { + url.scheme() == "https" + && url.host_str().is_some_and(|host| { + allowed_domains + .iter() + .any(|allowed| allowed.as_ref().eq_ignore_ascii_case(host)) + }) +} + +/// Checks the configuration rules core owns for every enabled provider +/// (spec §12.1): the endpoint is an `https` URL whose host is in +/// `allowed_domains`, it claims at least one source, every source has an +/// `[[ec.partners]]` entry, no source is claimed by two enabled providers, and +/// every enabled provider has a non-empty id of its own. +/// +/// Unknown kinds and capability checks need the provider crate; the adapter +/// builds the coordinators at startup and fails to start on either. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError::Invalid`] naming the first offending +/// provider. +pub fn validate_enabled_configs<'a>( + configs: impl IntoIterator, + partners: &PartnerRegistry, +) -> Result<(), Report> { + let mut claimed: HashSet = HashSet::new(); + let mut ids: HashSet<&str> = HashSet::new(); + for config in configs.into_iter().filter(|config| config.enabled) { + // The id keys UID ownership, opt-outs and the rate-limit bucket in + // the row, so two providers sharing one would read, refresh and clear + // each other's IDs. + let id = config.id.trim(); + if id.is_empty() || !ids.insert(id) { + return Err(Report::new(ProviderConfigError::Invalid).attach(format!( + "provider id `{}` is empty or used by another enabled provider", + config.id + ))); + } + let endpoint = Url::parse(&config.endpoint) + .change_context(ProviderConfigError::Invalid) + .attach_with(|| format!("endpoint for provider `{}` is not a URL", config.id))?; + if !destination_allowed(&endpoint, &config.allowed_domains) { + return Err(Report::new(ProviderConfigError::Invalid).attach(format!( + "endpoint for provider `{}` must be https with a host in allowed_domains", + config.id + ))); + } + let sources = parse_sources(config)?; + if sources.is_empty() { + return Err(Report::new(ProviderConfigError::Invalid) + .attach(format!("provider `{}` claims no sources", config.id))); + } + for source in sources { + if partners.find_by_source_domain(source.as_str()).is_none() { + return Err(Report::new(ProviderConfigError::Invalid).attach(format!( + "source `{source}` of provider `{}` has no [[ec.partners]] entry", + config.id + ))); + } + if !claimed.insert(source.clone()) { + return Err(Report::new(ProviderConfigError::Invalid).attach(format!( + "source `{source}` of provider `{}` is claimed by another enabled provider", + config.id + ))); + } + } + } + Ok(()) +} + +/// Fails when `[ec]` enables an identity provider on an adapter that has no +/// provider implementations. +/// +/// Ignoring the provider would leave its sources open to browser ingest with +/// no sign that it never runs. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Configuration`] naming the first enabled +/// provider. +pub fn reject_enabled_providers(ec: &Ec, adapter: &str) -> Result<(), Report> { + match ec.identity_providers.iter().find(|config| config.enabled) { + Some(config) => Err(Report::new(TrustedServerError::Configuration { + message: format!( + "identity provider `{}` is enabled, but the {adapter} adapter does not run identity providers", + config.id + ), + })), + None => Ok(()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde::Deserialize; + use serde_json::json; + + struct Fixed(ProviderCapabilities); + + impl IdentityProvider for Fixed { + fn capabilities(&self) -> &ProviderCapabilities { + &self.0 + } + + fn build_request( + &self, + _lookup: &LookupInput<'_>, + ) -> Result> { + Err(Report::new(ProviderError::Request)) + } + + fn parse_response<'body>( + &self, + _response: ProviderResponse<'body>, + ) -> Result, Report> { + Ok(LookupOutcome::NoMatch) + } + } + + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct FixedSettings { + max_eids: usize, + } + + struct FixedKind; + + impl ProviderKind for FixedKind { + const KIND: &'static str = "fixed"; + type Settings = FixedSettings; + type Provider = Fixed; + + fn build( + common: &ProviderCommon<'_>, + settings: FixedSettings, + ) -> Result> { + let max_eids = NonZeroUsize::new(settings.max_eids) + .ok_or_else(|| Report::new(ProviderConfigError::Invalid))?; + Ok(Fixed(ProviderCapabilities { + sources: common + .sources + .iter() + .map(|source| SourceCapability { + source: source.clone(), + input: SourceInput::Anonymous, + }) + .collect(), + shape: EidShape { + max_eids, + max_uids_per_eid: max_eids, + eid_level_fields: EidLevelFields::Preserved, + }, + })) + } + } + + fn config(settings: &serde_json::Value) -> IdentityProviderConfig { + serde_json::from_value(json!({ + "id": "example-provider", + "kind": FixedKind::KIND, + "endpoint": "https://ids.example.com/v1", + "allowed_domains": ["ids.example.com"], + "sources": ["ids.example.com"], + "settings": settings, + })) + .expect("should parse provider config") + } + + fn partners() -> PartnerRegistry { + let partner = serde_json::from_value(json!({ + "name": "ids", "source_domain": "ids.example.com" + })) + .expect("should build partner"); + PartnerRegistry::from_config(&[partner]).expect("should build registry") + } + + fn assert_invalid(error: &Report) { + assert!( + matches!(error.current_context(), ProviderConfigError::Invalid), + "should report invalid config" + ); + } + + #[test] + fn build_kind_parses_typed_settings_and_boxes_provider() { + let provider = build_kind::( + &config(&json!({ "max_eids": 2 })), + &ProviderSecrets::default(), + ) + .expect("should build"); + + assert_eq!( + provider.capabilities().sources.len(), + 1, + "should claim configured source" + ); + assert_eq!( + provider.capabilities().shape.max_eids.get(), + 2, + "should apply typed settings" + ); + } + + #[test] + fn build_kind_rejects_unknown_settings() { + let error = build_kind::( + &config(&json!({ "unexpected": true })), + &ProviderSecrets::default(), + ) + .err() + .expect("should reject"); + + assert_invalid(&error); + } + + #[test] + fn config_parses_from_toml_with_a_settings_table() { + let parsed: IdentityProviderConfig = toml::from_str( + "id = \"p\"\nkind = \"fixed\"\nendpoint = \"https://ids.example.com/\"\n[settings]\nmax_eids = 3", + ) + .expect("should parse TOML config"); + + assert_eq!(parsed.kind(), "fixed", "should keep kind"); + assert!(parsed.enabled, "should default to enabled"); + assert_eq!( + parsed.settings.get("max_eids"), + Some(&json!(3)), + "should keep provider settings" + ); + } + + #[test] + fn request_debug_hides_query_and_headers() { + let mut headers = http::HeaderMap::new(); + headers.insert( + http::header::AUTHORIZATION, + http::HeaderValue::from_static("Bearer example-secret"), + ); + let request = ProviderRequest { + method: http::Method::GET, + url: Url::parse("https://ids.example.com/v1?token=example-secret") + .expect("should parse"), + headers, + body: bytes::Bytes::new(), + context: None, + }; + + let printed = format!("{request:?}"); + + assert!( + !printed.contains("example-secret"), + "should not print secrets" + ); + assert!(printed.contains("ids.example.com"), "should print host"); + } + + #[test] + fn response_debug_hides_body_and_context() { + let headers = http::HeaderMap::new(); + let response = ProviderResponse { + status: http::StatusCode::OK, + headers: &headers, + body: b"example-uid", + context: Some("example-key"), + }; + + let printed = format!("{response:?}"); + + assert!( + !printed.contains("example-uid"), + "should not print the body" + ); + assert!( + !printed.contains("example-key"), + "should not print the context" + ); + assert!(printed.contains("200"), "should print the status"); + } + + #[test] + fn validate_enabled_configs_accepts_valid_config() { + validate_enabled_configs([&config(&json!({}))], &partners()) + .expect("should accept valid config"); + } + + #[test] + fn validate_enabled_configs_rejects_endpoint_outside_allowlist() { + let mut outside = config(&json!({})); + outside.endpoint = "https://other.example.com/".to_owned(); + let mut plain_http = config(&json!({})); + plain_http.endpoint = "http://ids.example.com/".to_owned(); + + for rejected in [outside, plain_http] { + let error = validate_enabled_configs([&rejected], &partners()) + .expect_err("should reject endpoint"); + assert_invalid(&error); + } + } + + #[test] + fn validate_enabled_configs_rejects_source_without_partner() { + let mut unknown = config(&json!({})); + unknown.sources = vec!["unknown.example.com".to_owned()]; + + let error = validate_enabled_configs([&unknown], &partners()) + .expect_err("should reject source without partner"); + + assert_invalid(&error); + } + + #[test] + fn validate_enabled_configs_rejects_source_claimed_twice() { + let first = config(&json!({})); + let mut second = config(&json!({})); + second.id = "second-provider".to_owned(); + + let error = validate_enabled_configs([&first, &second], &partners()) + .expect_err("should reject double claim"); + + assert_invalid(&error); + } + + #[test] + fn validate_enabled_configs_rejects_a_provider_without_sources() { + let mut empty = config(&json!({})); + empty.sources = Vec::new(); + + let error = validate_enabled_configs([&empty], &partners()) + .expect_err("should reject a provider that claims no sources"); + + assert_invalid(&error); + } + + #[test] + fn validate_enabled_configs_rejects_shared_or_empty_id() { + let registry = PartnerRegistry::from_config(&[ + serde_json::from_value(json!({ + "name": "ids", "source_domain": "ids.example.com" + })) + .expect("should build partner"), + serde_json::from_value(json!({ + "name": "other", "source_domain": "other.example.com" + })) + .expect("should build partner"), + ]) + .expect("should build registry"); + let first = config(&json!({})); + let mut shared = config(&json!({})); + shared.sources = vec!["other.example.com".to_owned()]; + let mut empty = config(&json!({})); + empty.id = " ".to_owned(); + + let error = validate_enabled_configs([&first, &shared], ®istry) + .expect_err("should reject a shared provider id"); + assert_invalid(&error); + let error = validate_enabled_configs([&empty], ®istry) + .expect_err("should reject an empty provider id"); + assert_invalid(&error); + } + + #[test] + fn reject_enabled_providers_fails_only_for_an_enabled_provider() { + let mut ec = crate::test_support::tests::create_test_settings().ec; + let mut disabled = config(&json!({})); + disabled.enabled = false; + ec.identity_providers = vec![disabled]; + reject_enabled_providers(&ec, "Axum").expect("should accept a disabled provider"); + + ec.identity_providers.push(config(&json!({}))); + let error = + reject_enabled_providers(&ec, "Axum").expect_err("should reject an enabled provider"); + + assert!( + matches!( + error.current_context(), + TrustedServerError::Configuration { .. } + ), + "should report a configuration error" + ); + } + + #[test] + fn validate_enabled_configs_ignores_disabled_providers() { + let first = config(&json!({})); + let mut disabled = config(&json!({})); + disabled.enabled = false; + disabled.endpoint = "https://other.example.com/".to_owned(); + + validate_enabled_configs([&first, &disabled], &partners()) + .expect("should ignore disabled provider"); + } + + #[test] + fn owned_entries_yields_only_configured_sources() { + let entry: KvEntry = serde_json::from_value(json!({ + "v": 1, + "created": 1, + "consent": { "ok": true, "updated": 1 }, + "geo": { "country": "US" }, + "ids": { + "ids.example.com": { "uid": "u-1", "eids": [{ "uids": [{ "id": "u-1", "atype": 3, "owner": "example-provider" }] }] }, + "other.example.com": { "uid": "u-2", "eids": [{ "uids": [{ "id": "u-2", "atype": 3, "owner": "example-provider" }] }] } + } + })) + .expect("should build entry"); + let sources = [SourceDomain::try_from("ids.example.com").expect("should parse")]; + let owner = ProviderId::new("example-provider"); + + let owned: Vec<&str> = OwnedEntries::new(Some(&entry), &sources, &owner) + .iter() + .map(|(source, _)| source) + .collect(); + + assert_eq!( + owned, + ["ids.example.com"], + "should yield owned sources only" + ); + } + + #[test] + fn empty_owned_entries_yields_nothing() { + assert_eq!( + OwnedEntries::empty().iter().count(), + 0, + "should yield no entries" + ); + } + + #[test] + fn owned_entries_hides_entries_another_provider_wrote() { + let entry: KvEntry = serde_json::from_value(json!({ + "v": 1, + "created": 1, + "consent": { "ok": true, "updated": 1 }, + "geo": { "country": "US" }, + "ids": { + "ids.example.com": { "uid": "u-1", "eids": [ + { "uids": [{ "id": "u-1", "atype": 3, "owner": "former-provider" }] }, + { "uids": [{ "id": "u-2", "atype": 3 }] } + ] } + } + })) + .expect("should build entry"); + let sources = [SourceDomain::try_from("ids.example.com").expect("should parse")]; + let owner = ProviderId::new("example-provider"); + + let count = OwnedEntries::new(Some(&entry), &sources, &owner) + .iter() + .count(); + + assert_eq!(count, 0, "should hide entries this provider did not write"); + } +} diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index 191e6c6d24..d5064a9842 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -560,6 +560,8 @@ mod tests { pull_sync_ttl_sec: ttl_sec, pull_sync_rate_limit: 20, ts_pull_token: Some(Redacted::new("token".to_owned())), + provider_owned: false, + owner_tcf_vendor_ids: Vec::new(), } } @@ -652,9 +654,7 @@ mod tests { if let EcKvSnapshot::Present { entry, .. } = &mut snapshot { entry.ids.insert( "ssp.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "uid".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("uid".to_owned()), ); } ec_context.set_kv_snapshot(snapshot); @@ -702,9 +702,7 @@ mod tests { entry.ids.insert( "b.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "uid-b".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("uid-b".to_owned()), ); assert!( @@ -801,12 +799,12 @@ mod tests { #[test] fn extract_pull_uid_rejects_oversized_uid() { - let long_uid = "x".repeat(513); + let long_uid = "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1); let body = format!("{{\"uid\":\"{long_uid}\"}}"); let response = make_response(200, body.as_bytes()); let uid = extract_pull_uid(response, "ssp.example.com"); - assert!(uid.is_none(), "should reject uid exceeding 512 bytes"); + assert!(uid.is_none(), "should reject uid exceeding MAX_UID_LENGTH"); } #[test] @@ -1189,7 +1187,7 @@ mod tests { let (entry, _) = graph .get(&ec_id) .expect("should read store") - .expect("tombstone should remain"); + .expect("should keep the tombstone"); assert!( entry.ids.is_empty(), "no partner UID may be written back onto a tombstoned row" @@ -1213,9 +1211,7 @@ mod tests { entry.consent.ok = true; entry.ids.insert( "alpha.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "already-known".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("already-known".to_owned()), ); graph .create(&ec_id, &entry) diff --git a/crates/trusted-server-core/src/ec/pull_sync_marker.rs b/crates/trusted-server-core/src/ec/pull_sync_marker.rs index 11df47296f..7cc86898b2 100644 --- a/crates/trusted-server-core/src/ec/pull_sync_marker.rs +++ b/crates/trusted-server-core/src/ec/pull_sync_marker.rs @@ -357,8 +357,8 @@ mod tests { fn settings_and_registry(domains: &[&str]) -> (Settings, PartnerRegistry) { let mut settings = create_test_settings(); settings.ec.partners = domains.iter().map(|domain| pull_partner(domain)).collect(); - let registry = PartnerRegistry::from_config(&settings.ec.partners) - .expect("should build pull partner registry"); + let registry = + PartnerRegistry::from_ec(&settings.ec).expect("should build pull partner registry"); (settings, registry) } @@ -375,9 +375,7 @@ mod tests { for domain in domains { entry.ids.insert( (*domain).to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: format!("uid-{domain}"), - }, + crate::ec::kv_types::KvPartnerId::single(format!("uid-{domain}")), ); } EcKvSnapshot::Present { @@ -510,9 +508,7 @@ mod tests { let mut tombstone = KvEntry::tombstone(1_000); tombstone.ids.insert( "a.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId { - uid: "stale".to_owned(), - }, + crate::ec::kv_types::KvPartnerId::single("stale".to_owned()), ); let snapshots = [ live_snapshot(EC_ID, &[]), diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index c4637b4310..4e840cd28c 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -5,12 +5,14 @@ //! lookup by source domain and API key hash. use std::collections::{HashMap, HashSet}; +use std::num::NonZeroU16; use error_stack::{Report, ResultExt as _}; +use crate::consent::{ConsentContext, tcf_vendor_consent_denied}; use crate::error::TrustedServerError; use crate::redacted::Redacted; -use crate::settings::EcPartner; +use crate::settings::{Ec, EcPartner}; use super::partner::{hash_api_key, normalize_partner_source_domain}; @@ -47,6 +49,24 @@ pub struct PartnerConfig { pub pull_sync_rate_limit: u32, /// Outbound bearer token for pull sync requests. pub ts_pull_token: Option>, + /// Whether an enabled identity provider claims this source. Browser + /// ingest skips it and batch sync rejects it. + pub provider_owned: bool, + /// TCF vendor IDs of the configured providers, enabled or disabled, that + /// claim this source. Its UIDs leave the server only while every one of + /// these vendors has consent. + pub owner_tcf_vendor_ids: Vec, +} + +impl PartnerConfig { + /// Whether the effective TCF consent denies any vendor that gates this + /// source. + #[must_use] + pub fn owner_vendor_consent_denied(&self, consent: &ConsentContext) -> bool { + self.owner_tcf_vendor_ids + .iter() + .any(|vendor| tcf_vendor_consent_denied(consent, *vendor)) + } } /// In-memory partner registry with O(1) lookups by source domain and API key hash. @@ -205,6 +225,59 @@ impl PartnerRegistry { }) } + /// Builds the registry from `[ec]` settings without provider ownership. + /// + /// For adapters with no identity-provider implementations, which refuse + /// to start with an enabled provider: no source is provider-owned, so + /// browser-supplied IDs are ingested for every source. Use + /// [`Self::from_ec_with_kinds`] where kinds are known. + /// + /// # Errors + /// + /// Returns the same errors as [`Self::from_config`]. + pub fn from_ec(ec: &Ec) -> Result> { + Self::from_config(&ec.partners) + } + + /// Builds the registry from `[ec]` settings and marks every source claimed + /// by an enabled identity provider whose kind is registered as + /// provider-owned. + /// + /// # Errors + /// + /// Returns the same errors as [`Self::from_config`], and + /// [`TrustedServerError::Configuration`] for an enabled provider whose + /// kind is not registered: it could never run, and its sources would + /// silently go back to browser ingest. + pub fn from_ec_with_kinds( + ec: &Ec, + is_registered_kind: impl Fn(&str) -> bool, + ) -> Result> { + let mut registry = Self::from_config(&ec.partners)?; + for provider in ec.identity_providers.iter().filter(|p| p.enabled) { + if !is_registered_kind(provider.kind()) { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "identity provider `{}` has kind `{}`, which this build does not include", + provider.id, + provider.kind() + ), + })); + } + for source in &provider.sources { + registry.mark_provider_owned(source, provider.tcf_vendor_id); + } + } + // A disabled provider's stored UIDs stay behind its vendor consent, + // though its sources go back to browser ingest. + for provider in ec.identity_providers.iter().filter(|p| !p.enabled) { + for source in &provider.sources { + registry.keep_vendor_gate(source, provider.tcf_vendor_id); + } + } + Ok(registry) + } + /// Returns an empty registry (no partners configured). #[must_use] pub fn empty() -> Self { @@ -234,12 +307,53 @@ impl PartnerRegistry { self.get(domain) } - /// Returns all partners with `pull_sync_enabled = true`. + /// Marks `source` as owned by an enabled identity provider, recording the + /// provider's TCF vendor ID, if any. + /// + /// Returns `false` when the source has no `[[ec.partners]]` entry. + pub fn mark_provider_owned(&mut self, source: &str, tcf_vendor_id: Option) -> bool { + let Ok(normalized) = normalize_partner_source_domain(source) else { + return false; + }; + match self.by_source_domain.get_mut(&normalized) { + Some(partner) => { + partner.provider_owned = true; + partner.owner_tcf_vendor_ids.extend(tcf_vendor_id); + true + } + None => false, + } + } + + /// Adds a disabled provider's TCF vendor ID to the gate for `source`, even + /// when an enabled provider now owns it, so egress still checks that + /// vendor's consent for the UIDs the disabled provider stored. + fn keep_vendor_gate(&mut self, source: &str, tcf_vendor_id: Option) { + let Ok(normalized) = normalize_partner_source_domain(source) else { + return; + }; + if let Some(partner) = self.by_source_domain.get_mut(&normalized) { + partner.owner_tcf_vendor_ids.extend(tcf_vendor_id); + } + } + + /// Whether an enabled identity provider owns `source`. + #[must_use] + pub fn is_provider_owned(&self, source: &str) -> bool { + self.find_by_source_domain(source) + .is_some_and(|partner| partner.provider_owned) + } + + /// Returns all partners with `pull_sync_enabled = true` whose source no + /// identity provider owns. + /// + /// A provider writes full-fidelity records for the sources it owns, so a + /// pull-sync write there would overwrite them with a single UID. #[must_use] pub fn pull_enabled_partners(&self) -> Vec<&PartnerConfig> { self.by_source_domain .values() - .filter(|p| p.pull_sync_enabled) + .filter(|p| p.pull_sync_enabled && !p.provider_owned) .collect() } @@ -305,6 +419,8 @@ fn build_partner_config( pull_sync_ttl_sec: partner.pull_sync_ttl_sec, pull_sync_rate_limit: partner.pull_sync_rate_limit, ts_pull_token: partner.ts_pull_token.clone(), + provider_owned: false, + owner_tcf_vendor_ids: Vec::new(), } } @@ -435,6 +551,133 @@ mod tests { } } + #[test] + fn from_ec_marks_enabled_provider_sources_only() { + let mut ec = crate::test_support::tests::create_test_settings().ec; + ec.partners = vec![ + make_partner("ids.example.com", &valid_api_token("ids-partner")), + make_partner("other.example.com", &valid_api_token("other-partner")), + ]; + ec.identity_providers = vec![ + toml::from_str( + "id = \"a\"\nkind = \"k\"\nendpoint = \"https://ids.example.com/\"\nsources = [\"ids.example.com\"]", + ) + .expect("should parse enabled provider"), + toml::from_str( + "id = \"b\"\nkind = \"k\"\nenabled = false\ntcf_vendor_id = 42\nendpoint = \"https://ids.example.com/\"\nsources = [\"other.example.com\"]", + ) + .expect("should parse disabled provider"), + ]; + + let registry = PartnerRegistry::from_ec_with_kinds(&ec, |kind| kind == "k") + .expect("should build registry"); + + assert!( + registry.is_provider_owned("ids.example.com"), + "should mark enabled provider source" + ); + assert!( + !registry.is_provider_owned("other.example.com"), + "should not mark disabled provider source" + ); + assert_eq!( + registry + .get("other.example.com") + .map(|partner| partner.owner_tcf_vendor_ids.clone()), + Some(vec![NonZeroU16::new(42).expect("should be non-zero")]), + "should keep the disabled provider's vendor consent gate" + ); + } + + #[test] + fn from_ec_with_kinds_keeps_a_disabled_providers_gate_on_a_reassigned_source() { + // Arrange: provider `old` (vendor 42) is disabled and its source is now + // claimed by enabled provider `new` (vendor 7, consented). + let mut ec = crate::test_support::tests::create_test_settings().ec; + ec.partners = vec![make_partner( + "ids.example.com", + &valid_api_token("ids-partner"), + )]; + ec.identity_providers = vec![ + toml::from_str( + "id = \"new\"\nkind = \"k\"\ntcf_vendor_id = 7\nendpoint = \"https://ids.example.com/\"\nsources = [\"ids.example.com\"]", + ) + .expect("should parse enabled provider"), + toml::from_str( + "id = \"old\"\nkind = \"k\"\nenabled = false\ntcf_vendor_id = 42\nendpoint = \"https://ids.example.com/\"\nsources = [\"ids.example.com\"]", + ) + .expect("should parse disabled provider"), + ]; + let consent = ConsentContext { + gdpr_applies: true, + tcf: Some(crate::consent::types::TcfConsent { + version: 2, + cmp_id: 1, + cmp_version: 1, + consent_screen: 0, + consent_language: "EN".to_owned(), + vendor_list_version: 1, + tcf_policy_version: 4, + created_ds: 0, + last_updated_ds: 0, + purpose_consents: vec![true; 24], + purpose_legitimate_interests: vec![false; 24], + vendor_consents: vec![7], + vendor_legitimate_interests: Vec::new(), + special_feature_opt_ins: vec![false; 12], + }), + ..ConsentContext::default() + }; + + // Act + let registry = PartnerRegistry::from_ec_with_kinds(&ec, |kind| kind == "k") + .expect("should build registry"); + let partner = registry + .get("ids.example.com") + .expect("should find the partner"); + + // Assert + assert!( + partner.provider_owned, + "should stay owned by the enabled provider" + ); + assert!( + partner.owner_vendor_consent_denied(&consent), + "should withhold the source while the disabled provider's vendor lacks consent" + ); + } + + #[test] + fn from_ec_with_kinds_rejects_an_unregistered_kind() { + let mut ec = crate::test_support::tests::create_test_settings().ec; + ec.partners = vec![make_partner( + "ids.example.com", + &valid_api_token("ids-partner"), + )]; + ec.identity_providers = vec![ + toml::from_str( + "id = \"a\"\nkind = \"missing\"\nendpoint = \"https://ids.example.com/\"\nsources = [\"ids.example.com\"]", + ) + .expect("should parse enabled provider"), + ]; + + let error = PartnerRegistry::from_ec_with_kinds(&ec, |kind| kind == "k") + .expect_err("should reject an enabled provider with an unregistered kind"); + let plain = PartnerRegistry::from_ec(&ec).expect("should build registry"); + + assert!( + matches!( + error.current_context(), + TrustedServerError::Configuration { .. } + ), + "should report a configuration error" + ); + assert!( + !plain.is_provider_owned("ids.example.com"), + "should not claim sources without provider implementations" + ); + } + #[test] fn empty_config_builds_empty_registry() { let registry = PartnerRegistry::from_config(&[]).expect("should build empty registry"); @@ -569,6 +812,25 @@ mod tests { assert!(result.is_err(), "should reject invalid source domain"); } + #[test] + fn pull_enabled_partners_excludes_provider_owned_sources() { + let mut pull_partner = make_partner("pull.example.com", &valid_api_token("token-p")); + pull_partner.api_token = None; + pull_partner.pull_sync_enabled = true; + pull_partner.pull_sync_url = Some("https://pull.example.com/sync".to_owned()); + pull_partner.pull_sync_allowed_domains = vec!["pull.example.com".to_owned()]; + pull_partner.ts_pull_token = Some(Redacted::new("outbound-token".to_owned())); + let mut registry = + PartnerRegistry::from_config(&[pull_partner]).expect("should build registry"); + + registry.mark_provider_owned("pull.example.com", None); + + assert!( + registry.pull_enabled_partners().is_empty(), + "should not pull sources an identity provider owns" + ); + } + #[test] fn pull_enabled_partners_filters_correctly() { let mut pull_partner = make_partner("pull.example.com", &valid_api_token("token-p")); diff --git a/crates/trusted-server-core/src/ec/write_plan.rs b/crates/trusted-server-core/src/ec/write_plan.rs new file mode 100644 index 0000000000..23718cd32c --- /dev/null +++ b/crates/trusted-server-core/src/ec/write_plan.rs @@ -0,0 +1,1195 @@ +//! Plans a navigation's single row change. +//! +//! Combines browser ingest (fill missing) and validated provider results +//! (replace the provider's sources, or remove them on an opt-out), prunes expired UIDs and enforces +//! [`MAX_ENTRY_BYTES`]. Pure: no I/O. +//! +//! Browser ingest skips provider-owned sources entirely (spec §8.4), so a +//! source the provider cleared on an opt-out is never refilled from the +//! browser's `ts-eids` cookie. + +use std::collections::btree_map; + +use super::EcKvSnapshot; +use super::coordinator::results::{LookupResults, LookupStatus, SkipReason}; +use super::eid_types::{SourceDomain, UnixSeconds}; +use super::kv::PartnerIdUpdate; +use super::kv_types::{KvEntry, KvPartnerId, MAX_ENTRY_BYTES}; +use super::page_load::{Generation, RowChange}; +use super::registry::PartnerRegistry; + +/// Inputs to [`plan_write`]. +#[derive(Debug, Clone, Copy)] +pub struct WritePlanInput<'a> { + /// Active EC ID. + pub ec_id: &'a str, + /// Row snapshot read for this request. + pub snapshot: &'a EcKvSnapshot, + /// The new row when this navigation creates one (new EC or recovery). + pub new_entry: Option<&'a KvEntry>, + /// Browser updates from `ts-eids` and `sharedId`, already reduced by the + /// ingest rules and consent-gated. + pub browser_ingest: &'a [PartnerIdUpdate], + /// Validated provider results. + pub lookups: &'a LookupResults, + /// Partner registry, for provider ownership. + pub registry: &'a PartnerRegistry, + /// Request time. + pub now: UnixSeconds, +} + +/// Plans the navigation's single row change. +/// +/// Returns `None` when nothing would change, when the row is a tombstone, or +/// when an update has no generation to guard it. +#[must_use] +pub fn plan_write(input: &WritePlanInput<'_>) -> Option { + let (base, expected) = match input.new_entry { + Some(entry) => (entry.clone(), None), + None => { + let entry = input.snapshot.entry_for(input.ec_id)?; + if !entry.consent.ok { + return None; + } + let Some(expected) = Generation::of(input.snapshot, input.ec_id) else { + log::info!( + "EC write planner: skipping update for '{}' (no_generation)", + super::log_id(input.ec_id) + ); + return None; + }; + (entry.clone(), Some(expected)) + } + }; + + // Prune first so an expired record frees its slot for browser ingest. + // Pruning alone never triggers a write; it rides along with a real change. + let mut entry = base; + for record in entry.ids.values_mut() { + record.prune_expired(input.now); + } + entry.prune_opt_outs(input.now); + entry.ids.retain(|_, record| !record.uid.is_empty()); + let pruned = entry.clone(); + + let lookups_apply = input.lookups.is_for(input.ec_id); + let ingested = apply_browser_ingest(&mut entry, input.browser_ingest, input.registry); + let written = if lookups_apply { + apply_lookups(&mut entry, input.lookups, input.now) + } else { + if !input.lookups.is_empty() { + log::info!( + "EC write planner: discarding lookups made for another EC ID than '{}'", + super::log_id(input.ec_id) + ); + } + Vec::new() + }; + let had_change = entry != pruned; + enforce_size_cap(&mut entry, &pruned, &written, &ingested); + // The cap may trim the only real change away; trimming alone never writes. + let changed = had_change && entry != pruned; + + match expected { + None => Some(RowChange::Create(entry)), + Some(_) if !changed => None, + Some(expected) => Some(RowChange::Update { entry, expected }), + } +} + +/// Fills missing sources from the browser, skipping provider-owned ones. +/// +/// Returns the sources it inserted, in insertion order. +fn apply_browser_ingest( + entry: &mut KvEntry, + updates: &[PartnerIdUpdate], + registry: &PartnerRegistry, +) -> Vec { + let mut ingested = Vec::new(); + for update in updates { + if registry.is_provider_owned(&update.partner_id) { + log::trace!( + "EC write planner: skipping browser UID for provider-owned source '{}'", + update.partner_id + ); + continue; + } + if let btree_map::Entry::Vacant(slot) = entry.ids.entry(update.partner_id.clone()) { + slot.insert(KvPartnerId::single(update.uid.clone())); + ingested.push(update.partner_id.clone()); + } + } + ingested +} + +fn apply_lookups( + entry: &mut KvEntry, + lookups: &LookupResults, + now: UnixSeconds, +) -> Vec { + let mut written = Vec::new(); + for lookup in lookups.iter() { + // A newer request may have recorded this provider's opt-out after + // the lookup was dispatched; never restore its UIDs over it. + if entry.opted_out(&lookup.provider, now) + && lookup.status != LookupStatus::Skipped(SkipReason::OptedOut) + { + log::info!( + "EC write planner: discarding results of opted-out provider '{}'", + lookup.provider + ); + continue; + } + for (source, eids) in &lookup.entries { + if let Some(record) = KvPartnerId::from_eids(eids.clone(), now) { + entry.ids.insert(source.as_str().to_owned(), record); + written.push(source.clone()); + } + } + if lookup.status == LookupStatus::Skipped(SkipReason::OptedOut) { + entry.remember_opt_out(&lookup.provider, now); + } + for source in &lookup.cleared { + if entry.ids.remove(source.as_str()).is_some() { + log::info!( + "EC write planner: removing '{source}' after provider '{}' opt-out", + lookup.provider + ); + } + } + } + written +} + +fn serialized_len(entry: &KvEntry) -> usize { + serde_json::to_vec(entry).map_or(usize::MAX, |bytes| bytes.len()) +} + +/// Serialized bytes one `ids` member adds to the row: its record, its +/// escaped key, the colon and a separating comma. +fn member_len(source: &str, record: &KvPartnerId) -> usize { + let key = serde_json::to_vec(source).map_or(usize::MAX, |bytes| bytes.len()); + let value = serde_json::to_vec(record).map_or(usize::MAX, |bytes| bytes.len()); + key.saturating_add(value).saturating_add(2) +} + +/// Shrinks `entry` until it serializes within [`MAX_ENTRY_BYTES`]. +/// +/// `base` is the stored row before this navigation's changes. A row that was +/// already over the cap (written before the cap existed) loses its largest +/// stored records first, so the change that triggered the write still lands. +/// This navigation's provider UIDs then compete with the stored row only, and +/// its browser UIDs, which the browser offers again, make way for them. A +/// provider refresh that no longer fits keeps the record it would replace +/// when that record fits. +/// +/// # Performance +/// +/// Tracks the row size by the bytes each drop removes and serializes the +/// whole row again only when that estimate reaches the cap, so the cost is +/// linear in the number of dropped UIDs and records. +fn enforce_size_cap( + entry: &mut KvEntry, + base: &KvEntry, + written: &[SourceDomain], + ingested: &[String], +) { + let mut size = serialized_len(entry); + if size <= MAX_ENTRY_BYTES { + return; + } + if serialized_len(base) > MAX_ENTRY_BYTES { + drop_largest_records(entry, &mut size, |source| { + !written.iter().any(|written| written.as_str() == source) + && !ingested.iter().any(|ingested| ingested == source) + }); + } + let browser_len: usize = ingested + .iter() + .filter_map(|source| { + entry + .ids + .get(source) + .map(|record| member_len(source, record)) + }) + .sum(); + // Drop this navigation's provider UIDs that do not fit, newest source + // last-in first. + for source in written.iter().rev() { + if size.saturating_sub(browser_len) <= MAX_ENTRY_BYTES { + break; + } + drop_provider_uids(entry, &mut size, base, source.as_str(), browser_len); + } + // Then this navigation's browser UIDs, newest first: the browser offers + // them again, while a stored record would be lost. + for source in ingested.iter().rev() { + if size <= MAX_ENTRY_BYTES { + break; + } + if let Some(record) = entry.ids.remove(source) { + log::info!( + "EC write planner: dropping browser UID for '{source}' to fit the row size cap" + ); + size = size.saturating_sub(member_len(source, &record)); + if size <= MAX_ENTRY_BYTES { + size = serialized_len(entry); + } + } + } + if size > MAX_ENTRY_BYTES { + drop_largest_records(entry, &mut size, |_| true); + if size > MAX_ENTRY_BYTES { + log::warn!("EC write planner: row exceeds size cap with no partner IDs left to drop"); + } + } +} + +/// Drops `source`'s provider UIDs, last first, until the row without this +/// navigation's browser UIDs (`browser_len`) fits [`MAX_ENTRY_BYTES`]. +/// +/// When every UID is dropped, the record `base` stored for `source` is put +/// back if it fits, so a refresh that no longer fits never deletes it. +fn drop_provider_uids( + entry: &mut KvEntry, + size: &mut usize, + base: &KvEntry, + source: &str, + browser_len: usize, +) { + let Some(record) = entry + .ids + .get_mut(source) + .filter(|record| record.is_provider_written()) + else { + return; + }; + while size.saturating_sub(browser_len) > MAX_ENTRY_BYTES && !record.uid.is_empty() { + let before = member_len(source, record); + record.drop_last_uid(); + if record.uid.is_empty() { + break; + } + *size = size.saturating_sub(before.saturating_sub(member_len(source, record))); + if size.saturating_sub(browser_len) <= MAX_ENTRY_BYTES { + *size = serialized_len(entry); + return; + } + } + if !record.uid.is_empty() { + return; + } + entry.ids.remove(source); + *size = serialized_len(entry); + if let Some(previous) = base.ids.get(source) { + let restored = size.saturating_add(member_len(source, previous)); + if restored.saturating_sub(browser_len) <= MAX_ENTRY_BYTES { + log::info!( + "EC write planner: keeping the stored record for '{source}' as its refresh does not fit the row size cap" + ); + entry.ids.insert(source.to_owned(), previous.clone()); + *size = serialized_len(entry); + } + } +} + +/// Drops whole records that `droppable` accepts, largest first, until `entry` +/// fits [`MAX_ENTRY_BYTES`] or none is left. +fn drop_largest_records(entry: &mut KvEntry, size: &mut usize, droppable: impl Fn(&str) -> bool) { + let mut by_size: Vec<(usize, String)> = entry + .ids + .iter() + .filter(|(source, _)| droppable(source)) + .map(|(source, record)| (member_len(source, record), source.clone())) + .collect(); + // A stable ascending sort pops the last of equally large records first. + by_size.sort_by_key(|(len, _)| *len); + while *size > MAX_ENTRY_BYTES { + let Some((len, largest)) = by_size.pop() else { + return; + }; + log::warn!("EC write planner: dropping '{largest}' to fit the row size cap"); + entry.ids.remove(&largest); + *size = size.saturating_sub(len); + if *size <= MAX_ENTRY_BYTES { + *size = serialized_len(entry); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ec::coordinator::results::{DropCounts, LookupStatus, ProviderLookup, SkipReason}; + use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, SourceDomain, UidValue}; + use crate::ec::kv::KvIdentityGraph; + use crate::ec::kv_types::{KvEid, KvPartnerId, KvUid, OPT_OUT_MEMORY_SECONDS}; + use crate::settings::EcPartner; + + const EC_ID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; + + fn partner(source: &str) -> EcPartner { + serde_json::from_value(serde_json::json!({ "name": source, "source_domain": source, "bidstream_enabled": true })) + .expect("should build partner") + } + + fn registry(owned: &[&str]) -> PartnerRegistry { + let mut registry = PartnerRegistry::from_config(&[ + partner("browser.example.com"), + partner("ids.example.com"), + ]) + .expect("should build registry"); + for source in owned { + assert!( + registry.mark_provider_owned(source, None), + "should mark {source}" + ); + } + registry + } + + fn live_entry() -> KvEntry { + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry + } + + fn present(entry: &KvEntry) -> EcKvSnapshot { + let graph = KvIdentityGraph::in_memory("plan"); + graph.create(EC_ID, entry).expect("should seed row"); + graph.load_snapshot(EC_ID) + } + + fn found(source: &str, ids: &[&str]) -> LookupResults { + let uids = ids + .iter() + .map(|id| { + KvUid::new( + UidValue::try_from(*id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::Never, + ) + }) + .collect(); + LookupResults::new( + EC_ID, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Found { + kept: 1, + dropped: DropCounts::default(), + }, + entries: vec![( + SourceDomain::try_from(source).expect("should build source"), + vec![KvEid::new(EidFields::default(), uids)], + )], + cleared: Vec::new(), + }], + ) + } + + fn input<'a>( + snapshot: &'a EcKvSnapshot, + new_entry: Option<&'a KvEntry>, + ingest: &'a [PartnerIdUpdate], + lookups: &'a LookupResults, + registry: &'a PartnerRegistry, + ) -> WritePlanInput<'a> { + WritePlanInput { + ec_id: EC_ID, + snapshot, + new_entry, + browser_ingest: ingest, + lookups, + registry, + now: UnixSeconds::new(2000), + } + } + + #[test] + fn new_ec_creates_row_with_ingest_and_lookups() { + let registry = registry(&["ids.example.com"]); + let snapshot = EcKvSnapshot::Missing { + ec_id: EC_ID.to_owned(), + }; + let new_entry = live_entry(); + let ingest = [ + PartnerIdUpdate::new("browser.example.com", "b-1"), + PartnerIdUpdate::new("ids.example.com", "client"), + ]; + let lookups = found("ids.example.com", &["p-1"]); + + let change = plan_write(&input( + &snapshot, + Some(&new_entry), + &ingest, + &lookups, + ®istry, + )) + .expect("should plan create"); + + let RowChange::Create(entry) = change else { + panic!("should create a new row") + }; + assert_eq!( + entry.ids["browser.example.com"].uid, "b-1", + "should ingest browser uid" + ); + assert_eq!( + entry.ids["ids.example.com"].uid, "p-1", + "should store provider uid, not client uid" + ); + } + + #[test] + fn lookups_for_another_ec_id_are_discarded() { + let registry = registry(&["ids.example.com"]); + let snapshot = EcKvSnapshot::Missing { + ec_id: EC_ID.to_owned(), + }; + let new_entry = live_entry(); + let lookups_for_old = LookupResults::new( + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.OLD123", + found("ids.example.com", &["p-1"]).iter().cloned().collect(), + ); + + let change = plan_write(&input( + &snapshot, + Some(&new_entry), + &[], + &lookups_for_old, + ®istry, + )) + .expect("should plan create"); + + let RowChange::Create(entry) = change else { + panic!("should create a new row") + }; + assert!( + !entry.ids.contains_key("ids.example.com"), + "should not attribute the old EC ID's lookups to the new row" + ); + } + + #[test] + fn unchanged_returning_row_plans_no_write() { + let registry = registry(&[]); + let mut entry = live_entry(); + entry + .ids + .insert("browser.example.com".to_owned(), KvPartnerId::single("b-1")); + let snapshot = present(&entry); + let ingest = [PartnerIdUpdate::new("browser.example.com", "b-2")]; + let lookups = LookupResults::default(); + + assert!( + plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)).is_none(), + "should not overwrite an existing browser uid" + ); + } + + #[test] + fn lookup_replaces_provider_source_with_cas_update() { + let registry = registry(&["ids.example.com"]); + let mut entry = live_entry(); + entry + .ids + .insert("ids.example.com".to_owned(), KvPartnerId::single("legacy")); + let snapshot = present(&entry); + let lookups = found("ids.example.com", &["p-1", "p-2"]); + + let change = plan_write(&input(&snapshot, None, &[], &lookups, ®istry)) + .expect("should plan update"); + + let RowChange::Update { entry, .. } = change else { + panic!("should update existing row") + }; + assert_eq!( + entry.ids["ids.example.com"].eids()[0].uids().len(), + 2, + "should keep both provider uids" + ); + } + + #[test] + fn tombstoned_row_plans_nothing() { + let registry = registry(&[]); + let snapshot = present(&KvEntry::tombstone(1000)); + let lookups = found("ids.example.com", &["p-1"]); + assert!( + plan_write(&input(&snapshot, None, &[], &lookups, ®istry)).is_none(), + "should never write over a tombstone" + ); + } + + #[test] + fn snapshot_without_generation_plans_nothing() { + let registry = registry(&[]); + let snapshot = EcKvSnapshot::Present { + ec_id: EC_ID.to_owned(), + entry: Box::new(live_entry()), + generation: None, + }; + let ingest = [PartnerIdUpdate::new("browser.example.com", "b-1")]; + let lookups = LookupResults::default(); + assert!( + plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)).is_none(), + "should not overwrite blindly" + ); + } + + #[test] + fn oversized_row_without_provider_uids_drops_records_until_it_fits() { + let registry = registry(&[]); + let mut entry = live_entry(); + for i in 0..40 { + entry.ids.insert( + format!("source{i:02}.example.com"), + KvPartnerId::single(format!("{i:03}{}", "x".repeat(500))), + ); + } + let snapshot = EcKvSnapshot::Missing { + ec_id: EC_ID.to_owned(), + }; + let lookups = LookupResults::default(); + + let change = plan_write(&input(&snapshot, Some(&entry), &[], &lookups, ®istry)) + .expect("should plan create"); + + let RowChange::Create(entry) = change else { + panic!("should create") + }; + let size = serde_json::to_vec(&entry).expect("should serialize").len(); + assert!( + size <= MAX_ENTRY_BYTES, + "should fit the row cap, got {size}" + ); + assert!(!entry.ids.is_empty(), "should keep the records that fit"); + } + + #[test] + fn oversized_row_drops_provider_uids_until_it_fits() { + let registry = registry(&["ids.example.com"]); + let snapshot = EcKvSnapshot::Missing { + ec_id: EC_ID.to_owned(), + }; + let new_entry = live_entry(); + let big: Vec = (0..40) + .map(|i| format!("{i:03}{}", "x".repeat(500))) + .collect(); + let ids: Vec<&str> = big.iter().map(String::as_str).collect(); + let lookups = found("ids.example.com", &ids); + + let change = plan_write(&input( + &snapshot, + Some(&new_entry), + &[], + &lookups, + ®istry, + )) + .expect("should plan create"); + + let RowChange::Create(entry) = change else { + panic!("should create") + }; + let size = serde_json::to_vec(&entry).expect("should serialize").len(); + assert!( + size <= MAX_ENTRY_BYTES, + "should fit the row cap, got {size}" + ); + let kept = entry.ids["ids.example.com"].eids()[0].uids(); + assert!(kept.len() < 40, "should drop some uids"); + assert_eq!(kept[0].id().as_str(), ids[0], "should drop from the end"); + } + + fn ingest_owned() -> [PartnerIdUpdate; 1] { + [PartnerIdUpdate::new("ids.example.com", "browser-id")] + } + + fn failed_lookup(status: LookupStatus) -> LookupResults { + LookupResults::new( + EC_ID, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status, + entries: Vec::new(), + cleared: Vec::new(), + }], + ) + } + + #[test] + fn opt_out_removes_the_provider_sources_and_blocks_browser_fallback() { + // Arrange + let registry = registry(&["ids.example.com"]); + let snapshot = present(&provider_entry(&["p-1"])); + let ingest = ingest_owned(); + let lookups = LookupResults::new( + EC_ID, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Skipped(SkipReason::OptedOut), + entries: Vec::new(), + cleared: vec![SourceDomain::try_from("ids.example.com").expect("should parse")], + }], + ); + + // Act + let change = plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)) + .expect("should plan update"); + + // Assert + let RowChange::Update { entry, .. } = change else { + panic!("should update existing row") + }; + assert!( + !entry.ids.contains_key("ids.example.com"), + "should remove the opted-out source without a browser fallback" + ); + } + + #[test] + fn opt_out_is_remembered_without_extending_it() { + // Arrange + let registry = registry(&["ids.example.com"]); + let ingest: [PartnerIdUpdate; 0] = []; + let provider = ProviderId::new("example-provider"); + let lookups = failed_lookup(LookupStatus::Skipped(SkipReason::OptedOut)); + + // Act: the first opt-out is remembered. + let first = plan_write(&input( + &present(&provider_entry(&["p-1"])), + None, + &ingest, + &lookups, + ®istry, + )) + .expect("should plan update"); + + // Assert + let RowChange::Update { entry, .. } = first else { + panic!("should update existing row") + }; + let until = entry.opt_outs[&provider]; + assert_eq!( + until, + 2000 + OPT_OUT_MEMORY_SECONDS, + "should remember the opt-out for the memory period" + ); + + // Act: a skip because of the remembered opt-out changes nothing. + let again = plan_write(&input(&present(&entry), None, &ingest, &lookups, ®istry)); + + // Assert + assert!(again.is_none(), "should not extend a remembered opt-out"); + } + + #[test] + fn lapsed_opt_out_alone_plans_no_write() { + let registry = registry(&["ids.example.com"]); + let ingest: [PartnerIdUpdate; 0] = []; + let mut entry = live_entry(); + entry + .opt_outs + .insert(ProviderId::new("example-provider"), 1000); + let lookups = LookupResults::default(); + + let change = plan_write(&input(&present(&entry), None, &ingest, &lookups, ®istry)); + + assert!(change.is_none(), "should not write only to prune"); + } + + #[test] + fn lapsed_opt_out_is_pruned_with_a_real_change() { + let registry = registry(&["ids.example.com"]); + let ingest = [PartnerIdUpdate::new("browser.example.com", "b-1")]; + let mut entry = live_entry(); + entry + .opt_outs + .insert(ProviderId::new("example-provider"), 1000); + let lookups = LookupResults::default(); + + let change = plan_write(&input(&present(&entry), None, &ingest, &lookups, ®istry)) + .expect("should plan update"); + + let RowChange::Update { entry, .. } = change else { + panic!("should update existing row") + }; + assert!(entry.opt_outs.is_empty(), "should forget a lapsed opt-out"); + } + + #[test] + fn not_due_skip_with_an_expired_uid_plans_no_write() { + let registry = registry(&["ids.example.com"]); + let ingest: [PartnerIdUpdate; 0] = []; + let mut entry = live_entry(); + let expired = KvUid::new( + UidValue::try_from("old-1").expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::At(UnixSeconds::new(1500)), + ); + entry.ids.insert( + "ids.example.com".to_owned(), + KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![expired])], + UnixSeconds::new(1000), + ) + .expect("should build provider record"), + ); + let lookups = LookupResults::new( + EC_ID, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Skipped(SkipReason::NotDue), + entries: Vec::new(), + cleared: Vec::new(), + }], + ); + + let change = plan_write(&input(&present(&entry), None, &ingest, &lookups, ®istry)); + + assert!( + change.is_none(), + "should not write only to prune an expired UID" + ); + } + + #[test] + fn browser_ingest_fills_a_slot_freed_by_an_expired_record() { + // The source is not provider-owned (its provider was disabled), so the + // browser may fill it once the expired provider record is pruned. + let registry = registry(&[]); + let ingest = [PartnerIdUpdate::new("ids.example.com", "browser-1")]; + let lookups = LookupResults::default(); + let mut entry = live_entry(); + let expired = KvUid::new( + UidValue::try_from("old-1").expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::At(UnixSeconds::new(1500)), + ); + entry.ids.insert( + "ids.example.com".to_owned(), + KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![expired])], + UnixSeconds::new(1000), + ) + .expect("should build provider record"), + ); + + let change = plan_write(&input(&present(&entry), None, &ingest, &lookups, ®istry)); + + let Some(RowChange::Update { entry, .. }) = change else { + panic!("should plan an update that fills the freed slot"); + }; + assert_eq!( + entry + .ids + .get("ids.example.com") + .map(|record| record.uid.as_str()), + Some("browser-1"), + "should store the browser UID in place of the expired record" + ); + } + + fn provider_entry(ids: &[&str]) -> KvEntry { + let mut entry = live_entry(); + let lookups = found("ids.example.com", ids); + let (_, eids) = lookups.iter().next().expect("should hold a lookup").entries[0].clone(); + entry.ids.insert( + "ids.example.com".to_owned(), + KvPartnerId::from_eids(eids, UnixSeconds::new(1000)).expect("should build record"), + ); + entry + } + + #[test] + fn browser_ingest_skips_a_provider_owned_source() { + // Arrange + let registry = registry(&["ids.example.com"]); + let snapshot = present(&live_entry()); + let ingest = ingest_owned(); + + for status in [ + LookupStatus::NoMatch, + LookupStatus::Timeout, + LookupStatus::Skipped(SkipReason::NotDue), + ] { + let lookups = failed_lookup(status); + + // Act + let change = plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)); + + // Assert + assert!( + change.is_none(), + "should not store a browser UID for a provider-owned source" + ); + } + } + + #[test] + fn opted_out_source_is_not_refilled_from_the_browser_on_the_next_navigation() { + // Arrange: the row after an opt-out holds nothing for the source. + let registry = registry(&["ids.example.com"]); + let snapshot = present(&live_entry()); + let ingest = ingest_owned(); + let lookups = LookupResults::default(); + + // Act + let change = plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)); + + // Assert + assert!( + change.is_none(), + "should not restore an opted-out provider UID from the browser cookie" + ); + } + + #[test] + fn new_row_skips_browser_uid_for_a_provider_owned_source() { + let registry = registry(&["ids.example.com"]); + let snapshot = EcKvSnapshot::Missing { + ec_id: EC_ID.to_owned(), + }; + let new_entry = live_entry(); + let ingest = ingest_owned(); + let lookups = LookupResults::default(); + + let change = plan_write(&input( + &snapshot, + Some(&new_entry), + &ingest, + &lookups, + ®istry, + )) + .expect("should plan create"); + + let RowChange::Create(entry) = change else { + panic!("should create a new row") + }; + assert!( + !entry.ids.contains_key("ids.example.com"), + "should not store a browser UID when creating the row" + ); + } + + #[test] + fn browser_fallback_is_not_stored_when_provider_found_the_source() { + let registry = registry(&["ids.example.com"]); + let snapshot = present(&live_entry()); + let ingest = ingest_owned(); + let lookups = found("ids.example.com", &["p-1"]); + + let change = plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)) + .expect("should plan update"); + + let RowChange::Update { entry, .. } = change else { + panic!("should update existing row") + }; + let record = &entry.ids["ids.example.com"]; + assert_eq!(record.uid, "p-1", "should store the provider uid"); + assert!(record.is_provider_written(), "should keep provider eids"); + } + + #[test] + fn browser_fallback_never_overwrites_a_live_provider_uid() { + let registry = registry(&["ids.example.com"]); + let snapshot = present(&provider_entry(&["p-1"])); + let ingest = ingest_owned(); + let lookups = failed_lookup(LookupStatus::NoMatch); + + assert!( + plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)).is_none(), + "should leave the live provider uid untouched" + ); + } + + #[test] + fn browser_fallback_does_not_overwrite_an_existing_fallback() { + let registry = registry(&["ids.example.com"]); + let mut entry = live_entry(); + entry + .ids + .insert("ids.example.com".to_owned(), KvPartnerId::single("older")); + let snapshot = present(&entry); + let ingest = ingest_owned(); + let lookups = failed_lookup(LookupStatus::NoMatch); + + assert!( + plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)).is_none(), + "should not overwrite an existing browser uid" + ); + } + + #[test] + fn later_provider_result_replaces_browser_fallback() { + let registry = registry(&["ids.example.com"]); + let mut entry = live_entry(); + entry.ids.insert( + "ids.example.com".to_owned(), + KvPartnerId::single("browser-id"), + ); + let snapshot = present(&entry); + let lookups = found("ids.example.com", &["p-1"]); + + let change = plan_write(&input( + &snapshot, + None, + &ingest_owned(), + &lookups, + ®istry, + )) + .expect("should plan update"); + + let RowChange::Update { entry, .. } = change else { + panic!("should update existing row") + }; + let record = &entry.ids["ids.example.com"]; + assert_eq!(record.uid, "p-1", "should replace the fallback"); + assert!(record.is_provider_written(), "should store provider eids"); + } + + fn over_cap_legacy_entry() -> KvEntry { + let mut entry = live_entry(); + for i in 0..40 { + entry.ids.insert( + format!("source{i:02}.example.com"), + KvPartnerId::single(format!("{i:03}{}", "x".repeat(500))), + ); + } + entry + } + + #[test] + fn over_cap_row_with_no_change_plans_no_write() { + // Arrange: a row written before the cap existed. + let registry = registry(&[]); + let entry = over_cap_legacy_entry(); + assert!( + serde_json::to_vec(&entry).expect("should serialize").len() > MAX_ENTRY_BYTES, + "should start over the cap" + ); + let lookups = LookupResults::default(); + let snapshot = EcKvSnapshot::Present { + ec_id: EC_ID.to_owned(), + entry: Box::new(entry), + generation: Some(1), + }; + + // Act + let change = plan_write(&input(&snapshot, None, &[], &lookups, ®istry)); + + // Assert + assert!( + change.is_none(), + "should not write when only the cap would trim" + ); + } + + fn over_cap_snapshot() -> EcKvSnapshot { + let entry = over_cap_legacy_entry(); + assert!( + serde_json::to_vec(&entry).expect("should serialize").len() > MAX_ENTRY_BYTES, + "should start over the cap" + ); + EcKvSnapshot::Present { + ec_id: EC_ID.to_owned(), + entry: Box::new(entry), + generation: Some(1), + } + } + + fn assert_fits(entry: &KvEntry) { + let size = serde_json::to_vec(entry).expect("should serialize").len(); + assert!( + size <= MAX_ENTRY_BYTES, + "should fit the row cap, got {size}" + ); + } + + #[test] + fn over_cap_row_keeps_this_navigations_browser_uid() { + // Arrange + let registry = registry(&[]); + let snapshot = over_cap_snapshot(); + let ingest = [PartnerIdUpdate::new("browser.example.com", "b-1")]; + let lookups = LookupResults::default(); + + // Act + let change = plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)); + + // Assert + let Some(RowChange::Update { entry, .. }) = change else { + panic!("should plan an update") + }; + assert_fits(&entry); + assert_eq!( + entry + .ids + .get("browser.example.com") + .map(|record| record.uid.as_str()), + Some("b-1"), + "should keep the browser UID that triggered the write" + ); + } + + #[test] + fn over_cap_row_keeps_this_navigations_provider_uid() { + // Arrange + let registry = registry(&["ids.example.com"]); + let snapshot = over_cap_snapshot(); + let lookups = found("ids.example.com", &["p-1"]); + + // Act + let change = plan_write(&input(&snapshot, None, &[], &lookups, ®istry)); + + // Assert + let Some(RowChange::Update { entry, .. }) = change else { + panic!("should plan an update") + }; + assert_fits(&entry); + assert_eq!( + entry + .ids + .get("ids.example.com") + .map(|record| record.uid.as_str()), + Some("p-1"), + "should keep the provider UID that triggered the write" + ); + } + + fn fill_to(mut entry: KvEntry, limit: usize) -> KvEntry { + let mut filler = 0; + loop { + let mut candidate = entry.clone(); + candidate.ids.insert( + format!("filler{filler:03}.example.com"), + KvPartnerId::single("f".repeat(100)), + ); + if serde_json::to_vec(&candidate) + .expect("should serialize") + .len() + > limit + { + return entry; + } + entry = candidate; + filler += 1; + } + } + + #[test] + fn provider_uid_wins_over_browser_uid_when_only_one_fits() { + // Arrange: room for either new UID alone, not both. + let registry = registry(&["ids.example.com"]); + let entry = fill_to(live_entry(), MAX_ENTRY_BYTES - 800); + let ingest = [PartnerIdUpdate::new("browser.example.com", "b".repeat(300))]; + let provider_uid = "p".repeat(300); + let lookups = found("ids.example.com", &[provider_uid.as_str()]); + + // Act + let change = plan_write(&input(&present(&entry), None, &ingest, &lookups, ®istry)); + + // Assert + let Some(RowChange::Update { entry, .. }) = change else { + panic!("should plan an update") + }; + assert_fits(&entry); + assert_eq!( + entry + .ids + .get("ids.example.com") + .map(|record| record.uid.as_str()), + Some(provider_uid.as_str()), + "should store the provider UID" + ); + assert!( + !entry.ids.contains_key("browser.example.com"), + "should drop the browser UID, which the browser offers again" + ); + } + + #[test] + fn refresh_that_no_longer_fits_keeps_the_stored_provider_record() { + // Arrange: the stored provider UID fits; its larger refresh does not. + let registry = registry(&["ids.example.com"]); + let entry = fill_to(provider_entry(&["p-old"]), MAX_ENTRY_BYTES - 300); + let fresh = "p".repeat(900); + let lookups = found("ids.example.com", &[fresh.as_str()]); + + // Act + let change = plan_write(&input(&present(&entry), None, &[], &lookups, ®istry)); + + // Assert + assert!( + change.is_none(), + "should keep the stored provider record rather than delete it" + ); + } + + #[test] + fn browser_ingest_never_evicts_a_stored_record_to_fit() { + // Arrange: a stored provider record plus fillers, just under the cap. + let registry = registry(&["ids.example.com"]); + let big: Vec = (0..18) + .map(|i| format!("{i:03}{}", "x".repeat(500))) + .collect(); + let ids: Vec<&str> = big.iter().map(String::as_str).collect(); + let mut entry = provider_entry(&ids); + let mut filler = 0; + loop { + let mut candidate = entry.clone(); + candidate.ids.insert( + format!("filler{filler:03}.example.com"), + KvPartnerId::single("f".repeat(100)), + ); + if serde_json::to_vec(&candidate) + .expect("should serialize") + .len() + > MAX_ENTRY_BYTES - 200 + { + break; + } + entry = candidate; + filler += 1; + } + let ingest = [PartnerIdUpdate::new("browser.example.com", "b".repeat(500))]; + let lookups = LookupResults::default(); + + // Act + let change = plan_write(&input(&present(&entry), None, &ingest, &lookups, ®istry)); + + // Assert: the browser UID is dropped, so nothing changes. + assert!( + change.is_none(), + "should drop the new browser UID rather than evict a stored record" + ); + } + + #[test] + fn found_result_never_restores_uids_over_a_newer_opt_out() { + // Arrange: another request recorded the opt-out after this lookup ran. + let registry = registry(&["ids.example.com"]); + let mut entry = live_entry(); + entry.opt_outs.insert( + ProviderId::new("example-provider"), + 2000 + OPT_OUT_MEMORY_SECONDS, + ); + let lookups = found("ids.example.com", &["p-restored"]); + + // Act + let change = plan_write(&input(&present(&entry), None, &[], &lookups, ®istry)); + + // Assert + assert!( + change.is_none(), + "should not write the opted-out provider's older result" + ); + } +} diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index c13260704f..33018ca519 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -7015,30 +7015,30 @@ external_bundle_sri = "sha384-AAAA" let provider = PrebidAuctionProvider::new(base_config()); let mut auction_request = create_test_auction_request(); auction_request.user.eids = Some(vec![ - crate::openrtb::Eid { - source: "liveramp.com".to_owned(), - uids: vec![crate::openrtb::Uid { + crate::openrtb::Eid::new( + "liveramp.com".to_owned(), + vec![crate::openrtb::Uid { id: "LR_xyz".to_owned(), atype: Some(3), ext: None, }], - }, - crate::openrtb::Eid { - source: "id5-sync.com".to_owned(), - uids: vec![crate::openrtb::Uid { + ), + crate::openrtb::Eid::new( + "id5-sync.com".to_owned(), + vec![crate::openrtb::Uid { id: "ID5_abc".to_owned(), atype: Some(1), ext: None, }], - }, - crate::openrtb::Eid { - source: "google.com".to_owned(), - uids: vec![crate::openrtb::Uid { + ), + crate::openrtb::Eid::new( + "google.com".to_owned(), + vec![crate::openrtb::Uid { id: "pair-id".to_owned(), atype: Some(571187), ext: None, }], - }, + ), ]); let settings = make_settings(); diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 15a6486fd8..0460984238 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -10,11 +10,8 @@ use matchit::Router; use crate::auction::AuctionPlan; use crate::constants::HEADER_X_TS_EC; -use crate::ec::EcContext; -use crate::ec::kv::KvIdentityGraph; use crate::error::TrustedServerError; use crate::geo::GeoInfo; -use crate::http_util::is_navigation_request; use crate::platform::RuntimeServices; use crate::settings::Settings; use crate::streaming_processor::StreamProcessor; @@ -786,15 +783,11 @@ impl IntegrationMetadata { /// Inputs to [`IntegrationRegistry::handle_proxy`]. /// -/// Bundled into a struct so the dispatch surface stays within the project's -/// 7-argument cap; `ec_context` and `req` participate in the borrow so the -/// whole thing shares one lifetime. +/// Bundled into a struct so the dispatch surface shares one lifetime. pub struct ProxyDispatchInput<'a> { pub method: &'a Method, pub path: &'a str, pub settings: &'a Settings, - pub kv: Option<&'a KvIdentityGraph>, - pub ec_context: &'a mut EcContext, pub services: &'a RuntimeServices, pub req: Request, } @@ -1018,7 +1011,8 @@ impl IntegrationRegistry { /// Dispatch a proxy request when an integration handles the path. /// /// This method removes any caller-supplied `x-ts-ec` before proxying. - /// Response-side cookie mutation is centralized in EC finalize. + /// Response-side cookie mutation is centralized in EC finalize. It never + /// mints an EC: only a publisher navigation may create the EC row. #[must_use] pub async fn handle_proxy( &self, @@ -1028,24 +1022,12 @@ impl IntegrationRegistry { method, path, settings, - kv, - ec_context, services, mut req, } = input; if let Some((proxy, _)) = self.find_route(method, path) { - // Organic proxy handler: generate if needed (best effort). - // Only generate for document navigations — subresource requests - // may lack consent signals such as the Sec-GPC header. - if is_navigation_request(&req) { - if let Err(err) = ec_context.generate_if_needed(settings, kv) { - log::warn!("EC generation failed for integration proxy: {err:?}"); - } - } else { - log::debug!( - "EC generation skipped for integration proxy: non-document request (path={path})", - ); - } + // Integration proxies never mint an EC: only the publisher + // navigation may create the row (one write per page load). // Remove any caller-supplied EC header rather than forwarding it. req.headers_mut().remove(HEADER_X_TS_EC.clone()); @@ -1835,14 +1817,10 @@ mod tests { .body(EdgeBody::empty()) .expect("should build request"); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); let response = futures::executor::block_on(registry.handle_proxy(ProxyDispatchInput { method: &http::Method::GET, path: "/integrations/test/echo", settings: &settings, - kv: None, - ec_context: &mut ec_context, services: &noop_services(), req, })) @@ -2189,8 +2167,6 @@ mod tests { HEADER_X_TS_EC.clone(), HeaderValue::from_static("some-ec-value"), ); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); let services = noop_services(); // Call handle_proxy (uses futures executor in test environment) @@ -2198,8 +2174,6 @@ mod tests { method: &Method::GET, path: "/integrations/test/ec", settings: &settings, - kv: None, - ec_context: &mut ec_context, services: &services, req, })); @@ -2239,8 +2213,6 @@ mod tests { HEADER_X_TS_EC.clone(), HeaderValue::from_static("evil;injected"), ); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); let services = crate::platform::test_support::noop_services(); @@ -2248,8 +2220,6 @@ mod tests { method: &Method::GET, path: "/integrations/test/ec", settings: &settings, - kv: None, - ec_context: &mut ec_context, services: &services, req, })) @@ -2288,8 +2258,6 @@ mod tests { )) .expect("should build Cookie header"), ); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); let services = crate::platform::test_support::noop_services(); @@ -2297,8 +2265,6 @@ mod tests { method: &Method::GET, path: "/integrations/test/ec", settings: &settings, - kv: None, - ec_context: &mut ec_context, services: &services, req, })) @@ -2334,8 +2300,6 @@ mod tests { HEADER_X_TS_EC.clone(), HeaderValue::from_static("some-ec-value"), ); - let mut ec_context = - EcContext::new_for_test(None, crate::consent::ConsentContext::default()); let services = crate::platform::test_support::noop_services(); @@ -2343,8 +2307,6 @@ mod tests { method: &Method::POST, path: "/integrations/test/ec", settings: &settings, - kv: None, - ec_context: &mut ec_context, services: &services, req, })); diff --git a/crates/trusted-server-core/src/openrtb.rs b/crates/trusted-server-core/src/openrtb.rs index 67cd7f6950..645031a7b0 100644 --- a/crates/trusted-server-core/src/openrtb.rs +++ b/crates/trusted-server-core/src/openrtb.rs @@ -70,10 +70,37 @@ pub struct ConsentedProvidersSettings { pub struct Eid { /// Identity provider domain (e.g. `"id5-sync.com"`). pub source: String, + /// `OpenRTB` `inserter`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub inserter: Option, + /// `OpenRTB` `matcher`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub matcher: Option, + /// `OpenRTB` `mm` (match method). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mm: Option, + /// EID-level extension data. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ext: Option, /// One or more user IDs from this provider. pub uids: Vec, } +impl Eid { + /// Builds an entry with no EID-level fields. + #[must_use] + pub fn new(source: impl Into, uids: Vec) -> Self { + Self { + source: source.into(), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids, + } + } +} + /// A single user identifier within an [`Eid`] entry. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -405,16 +432,46 @@ mod tests { ); } + #[test] + fn eid_serializes_eid_level_fields_only_when_present() { + let plain = Eid::new("ids.example.com", vec![]); + assert_eq!( + serde_json::to_value(&plain).expect("should serialize"), + serde_json::json!({ "source": "ids.example.com", "uids": [] }), + "should omit absent EID-level fields" + ); + let full = Eid { + inserter: Some("inserter.example.com".to_owned()), + matcher: Some("matcher.example.com".to_owned()), + mm: Some(3), + ext: Some(serde_json::json!({ "k": 1 })), + ..Eid::new("ids.example.com", vec![]) + }; + let value = serde_json::to_value(&full).expect("should serialize"); + assert_eq!( + value["inserter"], + serde_json::json!("inserter.example.com"), + "should write inserter" + ); + assert_eq!(value["mm"], serde_json::json!(3), "should write mm"); + let back: Eid = serde_json::from_value(value).expect("should accept EID-level fields"); + assert_eq!( + back.matcher.as_deref(), + Some("matcher.example.com"), + "should read matcher" + ); + } + #[test] fn eid_serializes_correctly() { - let eid = Eid { - source: "id5-sync.com".to_owned(), - uids: vec![Uid { + let eid = Eid::new( + "id5-sync.com".to_owned(), + vec![Uid { id: "ID5-abc123".to_owned(), atype: Some(1), ext: None, }], - }; + ); let serialized = serde_json::to_value(&eid).expect("should serialize"); assert_eq!(serialized["source"], "id5-sync.com", "source should match"); @@ -431,14 +488,14 @@ mod tests { #[test] fn eid_serializes_vendor_specific_atype() { - let eid = Eid { - source: "google.com".to_owned(), - uids: vec![Uid { + let eid = Eid::new( + "google.com".to_owned(), + vec![Uid { id: "pair-id".to_owned(), atype: Some(571187), ext: None, }], - }; + ); let serialized = serde_json::to_value(&eid).expect("should serialize"); diff --git a/crates/trusted-server-core/src/platform/http.rs b/crates/trusted-server-core/src/platform/http.rs index e1fe8ff1f2..a93753653e 100644 --- a/crates/trusted-server-core/src/platform/http.rs +++ b/crates/trusted-server-core/src/platform/http.rs @@ -414,8 +414,9 @@ pub trait PlatformHttpClient: Send + Sync { /// Collects an available response, waiting only until an absolute deadline. /// - /// On expiry, implementations take one nonblocking snapshot of complete - /// available responses before relinquishing unfinished requests. This + /// On expiry, implementations take one final snapshot of complete + /// available responses before relinquishing unfinished requests; it may + /// wait briefly for bytes the host already received. This /// preserves buffered bids when a publisher origin delays collection. /// Subsequent calls may drain only the completed snapshot. The deadline /// bounds network waiting, not eligibility by an upstream arrival timestamp. @@ -428,7 +429,7 @@ pub trait PlatformHttpClient: Send + Sync { /// /// Complete response bodies are buffered within adapter-specific limits. /// Deadline expiry drops incomplete bodies and pending handles after the - /// nonblocking snapshot; complete buffered responses remain collectible. + /// final snapshot; complete buffered responses remain collectible. /// /// # Examples /// diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 8d26af7859..af6484981f 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -41,7 +41,8 @@ use sha2::Digest as _; use web_time::Instant; use crate::auction::endpoints::{ - merge_auction_eids, resolve_auction_eids, resolve_client_auction_eids, + drop_vendor_denied_client_eids, merge_auction_eids, resolve_auction_eids, + resolve_client_auction_eids, }; use crate::auction::formats::sanitize_publisher_page_url; use crate::auction::orchestrator::{ @@ -3448,16 +3449,32 @@ fn request_head_snapshot(req: &Request) -> Request { struct EcSnapshotPreloadInput { is_navigation: bool, is_get: bool, + has_navigation_permit: bool, has_ec_id: bool, has_kv: bool, marker_valid: bool, auction_needs_row: bool, eid_cookie_may_need_persistence: bool, privacy_needs_row: bool, + snapshot_already_read: bool, } fn should_preload_ec_snapshot(input: &EcSnapshotPreloadInput) -> bool { + // An earlier step of this request already settled the row snapshot + // (identity-provider dispatch, or EC generation's authoritative miss), so + // a read adds nothing. + if input.snapshot_already_read { + return false; + } let eligible = input.is_navigation && input.is_get && input.has_ec_id && input.has_kv; + // Without a permit (no Fetch Metadata, or a prefetch) the navigation + // cannot write, so browser EID persistence no longer needs the row; the + // auction, a withdrawal and pull sync (unless the marker proves it + // complete) still do. + if !input.has_navigation_permit { + return eligible + && (input.auction_needs_row || input.privacy_needs_row || !input.marker_valid); + } let marker_can_skip = input.marker_valid && !input.auction_needs_row && !input.eid_cookie_may_need_persistence @@ -4956,9 +4973,15 @@ pub async fn handle_publisher_request( // explicit withdrawal `ec_allowed` is false, so auction dispatch forwards no // EC while the origin-overlapped snapshot read still happens for the active // ID, keeping the withdrawal CAS off the post-origin latency path. + // + // An EC minted on this request has no row until finalize creates it, and + // that create can still fail or be skipped (a redirect), so it is not sent + // to bidders until a later request proves it was issued. let active_ec_id_owned = ec_context.ec_value().map(str::to_owned); let active_ec_id = active_ec_id_owned.as_deref(); - let ec_id_owned = active_ec_id_owned.clone().filter(|_| ec_allowed); + let ec_id_owned = active_ec_id_owned + .clone() + .filter(|_| ec_allowed && !ec_context.ec_generated()); let ec_id = ec_id_owned.as_deref(); let cookie_jar = handle_request_cookies(&req)?; if let Some(registry) = auction.registry { @@ -5298,6 +5321,7 @@ pub async fn handle_publisher_request( let should_preload_ec = should_preload_ec_snapshot(&EcSnapshotPreloadInput { is_navigation, is_get, + has_navigation_permit: ec_context.has_navigation_permit(), has_ec_id: active_ec_id.is_some(), has_kv: kv.is_some(), marker_valid: ec_context.pull_sync_marker().is_valid(), @@ -5307,6 +5331,12 @@ pub async fn handle_publisher_request( .is_some_and(|registry| !registry.is_empty()), eid_cookie_may_need_persistence, privacy_needs_row: crate::ec::consent::ec_consent_withdrawn(&consent_context), + // A freshly generated EC already holds an authoritative `Missing` + // snapshot: its row is only created by this navigation's write. + snapshot_already_read: !matches!( + ec_context.kv_snapshot(), + crate::ec::EcKvSnapshot::NotRead + ), }); let mut pending_origin = None; // A shared-template hit skips the origin entirely, and an in-flight request @@ -5349,18 +5379,9 @@ pub async fn handle_publisher_request( ); } if should_preload_ec && let (Some(graph), Some(active_ec_id)) = (kv, active_ec_id) { - let refreshed = graph.load_snapshot(active_ec_id); - // Never downgrade an in-request Add-confirmed Present snapshot: a - // freshly created row can read back Missing/Failed on an - // eventually-consistent store, and rotating or suppressing that - // just-generated identity would fragment it. Adopt the refresh only - // when it keeps or upgrades to a Present row (the intended - // generation-refresh) — otherwise retain the confirmed entry. - let keep_present = ec_context.kv_snapshot().entry_for(active_ec_id).is_some() - && refreshed.entry_for(active_ec_id).is_none(); - if !keep_present { - ec_context.set_kv_snapshot(refreshed); - } + // A new EC's row is created only in finalize, so no snapshot read + // before this point can be an in-request confirmed row to protect. + ec_context.set_kv_snapshot(graph.load_snapshot(active_ec_id)); } // Dispatch SSP bid requests while req still has the original client headers @@ -6151,6 +6172,11 @@ fn apply_auction_eids_and_device( targeting.partner_registry, targeting.ec_context, ); + let client_eids = drop_vendor_denied_client_eids( + client_eids, + targeting.partner_registry, + targeting.ec_context.consent(), + ); let merged_eids = merge_auction_eids(client_eids, kv_eids); let had_eids = merged_eids.as_ref().is_some_and(|v| !v.is_empty()); auction_request.user.eids = @@ -8058,6 +8084,8 @@ mod tests { use flate2::read::GzDecoder; use flate2::write::GzEncoder; + use base64::Engine as _; + use super::*; use crate::auction::orchestrator::OrchestrationResult; use crate::auction::provider::{AuctionProvider, ProviderRequestOutcome}; @@ -8382,12 +8410,14 @@ mod tests { let baseline = EcSnapshotPreloadInput { is_navigation: true, is_get: true, + has_navigation_permit: true, has_ec_id: true, has_kv: true, marker_valid: false, auction_needs_row: false, eid_cookie_may_need_persistence: false, privacy_needs_row: false, + snapshot_already_read: false, }; assert!(should_preload_ec_snapshot(&baseline)); assert!(!should_preload_ec_snapshot(&EcSnapshotPreloadInput { @@ -8406,6 +8436,55 @@ mod tests { has_kv: false, ..baseline })); + assert!( + !should_preload_ec_snapshot(&EcSnapshotPreloadInput { + snapshot_already_read: true, + ..baseline + }), + "should skip the preload when the snapshot was already read" + ); + } + + #[test] + fn ec_snapshot_preload_without_a_permit_ignores_browser_eid_persistence() { + let no_permit = EcSnapshotPreloadInput { + is_navigation: true, + is_get: true, + has_navigation_permit: false, + has_ec_id: true, + has_kv: true, + marker_valid: true, + auction_needs_row: false, + eid_cookie_may_need_persistence: true, + privacy_needs_row: false, + snapshot_already_read: false, + }; + + assert!( + !should_preload_ec_snapshot(&no_permit), + "should not read a row the navigation can never write" + ); + assert!( + should_preload_ec_snapshot(&EcSnapshotPreloadInput { + auction_needs_row: true, + ..no_permit + }), + "should read the row for the auction's stored EIDs" + ); + assert!( + should_preload_ec_snapshot(&EcSnapshotPreloadInput { + privacy_needs_row: true, + ..no_permit + }), + "should read the row for a withdrawal" + ); + assert!( + should_preload_ec_snapshot(&EcSnapshotPreloadInput { + marker_valid: false, + ..no_permit + }), + "should read the row for pull sync while the marker is not valid" + ); } #[test] @@ -8413,12 +8492,14 @@ mod tests { let marker_only = EcSnapshotPreloadInput { is_navigation: true, is_get: true, + has_navigation_permit: true, has_ec_id: true, has_kv: true, marker_valid: true, auction_needs_row: false, eid_cookie_may_need_persistence: false, privacy_needs_row: false, + snapshot_already_read: false, }; assert!(!should_preload_ec_snapshot(&marker_only)); assert!(should_preload_ec_snapshot(&EcSnapshotPreloadInput { @@ -8683,6 +8764,7 @@ mod tests { scheduling_consent(), Some("203.0.113.7".to_owned()), ); + ec_context.set_navigation_permit(crate::ec::page_load::NavigationPermit::for_test()); assert!( ec_context.ec_allowed() && ec_context.ec_value().is_some(), "test precondition: an active, consent-allowed EC must exist" @@ -8843,8 +8925,8 @@ mod tests { create_test_settings() }; settings.ec.partners = vec![marker_partner("ssp.example.com")]; - let registry = PartnerRegistry::from_config(&settings.ec.partners) - .expect("should build pull partner registry"); + let registry = + PartnerRegistry::from_ec(&settings.ec).expect("should build pull partner registry"); let http = Arc::new(StubHttpClient::new()); http.set_concurrent_fanout(true); http.push_response(200, b"ok".to_vec()); @@ -8893,6 +8975,7 @@ mod tests { scheduling_consent(), Some("203.0.113.7".to_owned()), ); + ec_context.set_navigation_permit(crate::ec::page_load::NavigationPermit::for_test()); ec_context.set_pull_sync_marker_for_test( crate::ec::pull_sync_marker::PullSyncMarkerState::from_cookie(Some(marker)), ); @@ -9981,6 +10064,100 @@ mod tests { } } + fn vendor_42_denied_consent() -> ConsentContext { + let mut purposes = vec![false; 24]; + purposes[0] = true; + purposes[3] = true; + ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::Gdpr, + gdpr_applies: true, + tcf: Some(crate::consent::types::TcfConsent { + version: 2, + cmp_id: 1, + cmp_version: 1, + consent_screen: 0, + consent_language: "EN".to_owned(), + vendor_list_version: 1, + tcf_policy_version: 4, + created_ds: 0, + last_updated_ds: 0, + purpose_consents: purposes, + purpose_legitimate_interests: vec![false; 24], + vendor_consents: Vec::new(), + vendor_legitimate_interests: Vec::new(), + special_feature_opt_ins: vec![false; 12], + }), + ..ConsentContext::default() + } + } + + #[test] + fn navigation_auction_drops_vendor_denied_cookie_eids() { + // Arrange: a provider with vendor 42 owns one source, consent denies it. + let partners: Vec = ["ids.example.com", "other.example.com"] + .iter() + .map(|source| { + serde_json::from_value(serde_json::json!({ + "name": source, "source_domain": source, "bidstream_enabled": true + })) + .expect("should build partner") + }) + .collect(); + let mut registry = PartnerRegistry::from_config(&partners).expect("should build registry"); + assert!( + registry.mark_provider_owned("ids.example.com", std::num::NonZeroU16::new(42)), + "should mark the provider source" + ); + let cookie_eids = serde_json::json!([ + {"source": "ids.example.com", "id": "owned-uid", "atype": 1}, + {"source": "other.example.com", "id": "other-uid", "atype": 1} + ]); + let mut jar = CookieJar::new(); + jar.add_original( + cookie::Cookie::build(( + COOKIE_TS_EIDS, + base64::engine::general_purpose::STANDARD + .encode(serde_json::to_vec(&cookie_eids).expect("should serialize")), + )) + .secure(true) + .build(), + ); + let ec_id = format!("{}.navvnd", "a".repeat(64)); + let ec_context = EcContext::new_for_test(Some(ec_id.clone()), vendor_42_denied_consent()); + let services = noop_services(); + let mut auction_request = test_auction_request(); + + // Act + apply_auction_eids_and_device( + &mut auction_request, + &AuctionEidTargeting { + cookie_jar: Some(&jar), + ec_id: Some(&ec_id), + kv_snapshot: &crate::ec::EcKvSnapshot::NotRead, + partner_registry: Some(®istry), + ec_context: &ec_context, + services: &services, + geo: None, + path_label: "Server-side", + }, + ); + + // Assert + let sources: Vec<&str> = auction_request + .user + .eids + .as_deref() + .unwrap_or_default() + .iter() + .map(|eid| eid.source.as_str()) + .collect(); + assert_eq!( + sources, + vec!["other.example.com"], + "should drop the cookie EID of the vendor-denied provider source" + ); + } + fn build_request(method: Method, uri: &str) -> HttpRequest { HttpRequest::builder() .method(method) @@ -13655,6 +13832,8 @@ mod tests { scheduling_consent(), Some("203.0.113.7".to_owned()), ); + ec_context + .set_navigation_permit(crate::ec::page_load::NavigationPermit::for_test()); let response = handle_publisher_request( &settings, &services, @@ -16285,6 +16464,74 @@ mod tests { } } + #[tokio::test] + async fn publisher_auction_withholds_an_ec_minted_on_this_request() { + let mut settings = cookie_policy_settings(Some(&["ab_bucket"]), None, true); + Arc::make_mut(&mut settings).auction.providers = + crate::auction_config_types::AuctionConfig::legacy_provider_map(&[ + SCHEDULING_PROVIDER, + ]); + let stub = Arc::new(StubHttpClient::new()); + let cache = Arc::new(MemoryTemplateCache::default()); + let services = services(Arc::clone(&stub), Arc::clone(&cache)); + let graph = KvIdentityGraph::in_memory("minted-ec-store"); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let partners = PartnerRegistry::empty(); + queue_shareable_html(&stub); + let minted = format!("{}.Mint01", "c".repeat(64)); + let mut ec_context = EcContext::new_for_test(None, scheduling_consent()); + ec_context.replace_with_generated( + minted.clone(), + crate::ec::EcKvSnapshot::Missing { + ec_id: minted.clone(), + }, + ); + let captured = Arc::new(Mutex::new(None)); + let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + orchestrator.register_provider(Arc::new(SchedulingCaptureProvider { + captured: Arc::clone(&captured), + http: Arc::clone(&stub), + lookups: Arc::new(AtomicUsize::new(0)), + })); + let orchestrator = Arc::new(orchestrator); + + let response = handle_publisher_request( + &settings, + &services, + Some(&graph), + &mut ec_context, + AuctionDispatch { + integration_registry: &IntegrationRegistry::empty_for_tests(), + orchestrator: &orchestrator, + slots: &[article_slot()], + registry: Some(&partners), + }, + cookie_policy_request(&[b"ab_bucket=A"]), + EdgeCacheHeader::SMaxageFallback, + ) + .await + .expect("should serve a first visit"); + let response = finalize_test_publisher_response( + response, + &settings, + &services, + ®istry, + orchestrator, + Finalizer::Buffered, + ) + .await; + let body = body_of(response).await; + + assert!(!body.is_empty(), "should render the page"); + let captured = captured.lock().expect("should lock captured auction"); + let auction = captured.as_ref().expect("should dispatch the auction"); + assert_eq!( + auction.request.user.id, None, + "should not send an EC whose row does not exist yet" + ); + } + #[tokio::test] async fn template_cookie_publisher_warm_variant_finalizes_ec_withdrawal() { for finalizer in [Finalizer::Streaming, Finalizer::Buffered] { @@ -16350,6 +16597,9 @@ mod tests { scheduling_consent() }; let mut ec_context = EcContext::new_for_test(Some(identity.clone()), consent); + // A publisher page load: only a navigation may write the tombstone. + ec_context + .set_navigation_permit(crate::ec::page_load::NavigationPermit::for_test()); assert_eq!( ec_context.ec_allowed(), !withdrawn, @@ -20045,11 +20295,13 @@ for (const mode of ['ready', 'queued', 'stale', 'inactive', 'throws']) { } fn active_ec_context() -> EcContext { - EcContext::new_for_test_with_ip( + let mut ec_context = EcContext::new_for_test_with_ip( Some(format!("{}.CkId01", "b".repeat(64))), scheduling_consent(), Some("203.0.113.7".to_owned()), - ) + ); + ec_context.set_navigation_permit(crate::ec::page_load::NavigationPermit::for_test()); + ec_context } #[tokio::test] diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 3150ead138..33544d77d0 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -7,6 +7,7 @@ use serde::{Deserialize, Deserializer, Serialize, de::DeserializeOwned}; use serde_json::Value as JsonValue; use sha2::{Digest as _, Sha256}; use std::collections::{HashMap, HashSet}; +use std::num::{NonZeroU16, NonZeroU32}; use std::ops::{Deref, DerefMut}; use std::str::FromStr; use std::sync::OnceLock; @@ -635,11 +636,95 @@ impl EcPartner { } } +/// One `[[ec.identity_providers]]` entry. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct IdentityProviderConfig { + /// Unique provider ID; recorded as the owner of the UIDs it writes. + pub id: String, + /// Implementation kind, matched by the provider crate's `build`. + kind: String, + /// Whether lookups run. A disabled provider owns no sources. + #[serde( + default = "IdentityProviderConfig::default_enabled", + deserialize_with = "from_value_or_str" + )] + pub enabled: bool, + /// Lookup endpoint; must be `https` with a host in `allowed_domains`. + pub endpoint: String, + /// Exact hosts the coordinator may call for this provider. + #[serde(default, deserialize_with = "vec_from_seq_or_map")] + pub allowed_domains: Vec, + /// Source domains this provider owns; each needs an `[[ec.partners]]` entry. + #[serde(default, deserialize_with = "vec_from_seq_or_map")] + pub sources: Vec, + /// TCF vendor ID that must have vendor consent, when set. + #[serde(default)] + pub tcf_vendor_id: Option, + /// When the provider's lookup runs on a navigation. + #[serde(default)] + pub collect: CollectTiming, + /// Most lookups per EC hash per hour, when set; unset means unlimited. + /// ECs behind one client IP share the budget. + /// + /// The Fastly limiter enforces this as a per-minute budget of at least + /// one, so values below 60 behave like 60 per hour. + #[serde(default)] + pub rate_limit_per_hour: Option, + /// Secret Store key references, resolved at startup. + #[serde(default)] + pub secrets: ProviderSecrets, + /// Provider-specific settings, parsed by the provider kind. + /// + /// Held as a JSON object because settings travel through the JSON app + /// config pipeline. + #[serde(default)] + pub settings: serde_json::Map, +} + +impl IdentityProviderConfig { + const fn default_enabled() -> bool { + true + } + + /// The configured implementation kind. + #[must_use] + pub fn kind(&self) -> &str { + &self.kind + } +} + +/// When an identity provider's lookup runs on a navigation. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum CollectTiming { + /// Before the response, within `identity_budget_ms`; results reach the + /// navigation's own write and response. + #[default] + Page, + /// After the response has been sent, within + /// `identity_after_response_budget_ms`; results reach the row in one + /// follow-up write and are served from the next page load. + AfterResponse, +} + +/// Secret values for one identity provider. +/// +/// `EdgeZero` secret paths address fixed field names, so V1 supports one +/// optional bearer token per provider. +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ProviderSecrets { + /// Secret Store key whose value is the provider's token. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub token: Option>, +} + /// Edge Cookie (EC) configuration. /// /// Mapped from the `[ec]` TOML section. Controls EC identity generation, /// KV store names, and partner registry. -#[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] +#[derive(Debug, Clone, Deserialize, Serialize, Validate)] #[serde(deny_unknown_fields)] pub struct Ec { /// Publisher passphrase used as HMAC key for EC generation. @@ -672,6 +757,65 @@ pub struct Ec { #[serde(default, deserialize_with = "vec_from_seq_or_map")] #[validate(nested)] pub partners: Vec, + + /// Shared time budget, in milliseconds, for all provider lookups on one + /// navigation. + /// + /// `Ec` denies unknown fields, so this and the other identity fields are + /// skipped from serialization at their defaults: a config blob without + /// providers then still loads in a runtime built before them. + #[serde( + default = "Ec::default_identity_budget_ms", + skip_serializing_if = "Ec::is_default_identity_budget_ms", + deserialize_with = "from_value_or_str" + )] + #[validate(range( + min = 1, + max = 1000, + message = "must be between 1 and 1000 milliseconds" + ))] + pub identity_budget_ms: u64, + + /// Time budget, in milliseconds, for lookups of providers with + /// `collect = "after_response"`, measured from their dispatch after the + /// response has been sent. + #[serde( + default = "Ec::default_identity_after_response_budget_ms", + skip_serializing_if = "Ec::is_default_identity_after_response_budget_ms", + deserialize_with = "from_value_or_str" + )] + #[validate(range( + min = 1, + max = 5000, + message = "must be between 1 and 5000 milliseconds" + ))] + pub identity_after_response_budget_ms: u64, + + /// Server-side identity providers. + #[serde( + default, + skip_serializing_if = "Vec::is_empty", + deserialize_with = "vec_from_seq_or_map" + )] + pub identity_providers: Vec, +} + +// Not derived: the identity budgets default to their serde defaults so a +// default `Ec` passes the budget range checks and serializes no identity keys. +impl Default for Ec { + fn default() -> Self { + Self { + passphrase: Redacted::default(), + ec_store: None, + pull_sync_concurrency: 0, + cluster_trust_threshold: 0, + cluster_recheck_secs: 0, + partners: Vec::new(), + identity_budget_ms: Self::default_identity_budget_ms(), + identity_after_response_budget_ms: Self::default_identity_after_response_budget_ms(), + identity_providers: Vec::new(), + } + } } impl Ec { @@ -702,6 +846,28 @@ impl Ec { 3600 } + /// Default shared identity lookup budget (200 ms). + #[must_use] + pub const fn default_identity_budget_ms() -> u64 { + 200 + } + + /// Default after-response identity lookup budget (2 s). + #[must_use] + pub const fn default_identity_after_response_budget_ms() -> u64 { + 2000 + } + + #[allow(clippy::trivially_copy_pass_by_ref)] + fn is_default_identity_budget_ms(value: &u64) -> bool { + *value == Self::default_identity_budget_ms() + } + + #[allow(clippy::trivially_copy_pass_by_ref)] + fn is_default_identity_after_response_budget_ms(value: &u64) -> bool { + *value == Self::default_identity_after_response_budget_ms() + } + /// Returns `true` if `passphrase` matches a known placeholder value /// (case-insensitive). #[must_use] @@ -3570,6 +3736,19 @@ impl Settings { )); } } + for provider in &self.ec.identity_providers { + if provider + .secrets + .token + .as_ref() + .is_some_and(|token| EcPartner::is_placeholder_api_token(token.expose())) + { + insecure_fields.push(format!( + "ec.identity_providers[{}].secrets.token", + provider.id + )); + } + } for handler in &self.handlers { if Handler::is_placeholder_password(handler.password.expose()) { insecure_fields.push(format!("handlers[{}].password", handler.path)); @@ -4321,6 +4500,7 @@ where #[cfg(test)] mod tests { use super::*; + use crate::auction::build_orchestrator; use crate::integrations::{ IntegrationRegistry, gpt::GptConfig, lockr::LockrConfig, nextjs::NextJsIntegrationConfig, @@ -4333,6 +4513,28 @@ mod tests { use std::collections::BTreeSet; use std::sync::Arc; + #[test] + fn default_ec_uses_the_identity_budget_defaults() { + let ec = Ec::default(); + + assert_eq!( + (ec.identity_budget_ms, ec.identity_after_response_budget_ms), + ( + Ec::default_identity_budget_ms(), + Ec::default_identity_after_response_budget_ms() + ), + "should match the serde defaults" + ); + let serialized = serde_json::to_value(&ec).expect("should serialize"); + assert!( + serialized.get("identity_budget_ms").is_none() + && serialized + .get("identity_after_response_budget_ms") + .is_none(), + "should serialize no identity keys" + ); + } + #[test] fn html_finalization_timeout_defaults_and_boundaries() { let parsed = Settings::from_toml(&crate_test_settings_str()) @@ -4413,6 +4615,53 @@ mod tests { ); } + /// `Ec` denies unknown fields, so a binary built before identity providers + /// existed must still accept a blob that configures none. + #[test] + fn default_identity_settings_are_omitted_for_rollback() { + let settings = + Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings"); + + let serialized = serde_json::to_value(&settings.ec).expect("should serialize ec settings"); + + for field in [ + "identity_budget_ms", + "identity_after_response_budget_ms", + "identity_providers", + ] { + assert!( + serialized.get(field).is_none(), + "should omit default {field} for older binaries: {serialized}" + ); + } + } + + #[test] + fn identity_budgets_are_range_checked() { + let settings = + Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings"); + for (page, after_response, invalid) in [ + (1, 1, false), + (1000, 5000, false), + (0, 2000, true), + (1001, 2000, true), + (200, 0, true), + (200, 5001, true), + ] { + let ec = Ec { + identity_budget_ms: page, + identity_after_response_budget_ms: after_response, + ..settings.ec.clone() + }; + + assert_eq!( + ec.validate().is_err(), + invalid, + "should enforce identity budget bounds for {page} and {after_response}" + ); + } + } + /// `DebugConfig` denies unknown fields, so a binary built before /// `sandbox_metrics_enabled` existed must still accept a default blob. /// That only holds while the flag is skipped during serialization. @@ -6948,6 +7197,31 @@ source_domain = "partner.example.com" .expect("should accept a realistic partner pull token"); } + #[test] + fn reject_placeholder_secrets_includes_identity_provider_tokens() { + let mut settings = + Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings"); + settings.publisher.proxy_secret = Redacted::new("unit-test-proxy-secret".to_owned()); + settings.ec.passphrase = Redacted::new("test-secret-key-32-bytes-minimum".to_owned()); + let mut provider: IdentityProviderConfig = toml::from_str( + "id = \"example-provider\"\nkind = \"example-kind\"\nendpoint = \"https://ids.example.com/\"", + ) + .expect("should parse provider config"); + provider.secrets.token = Some(Redacted::new( + "partner-api-token-32-bytes-minimum".to_owned(), + )); + settings.ec.identity_providers = vec![provider]; + + let err = settings + .reject_placeholder_secrets() + .expect_err("should reject placeholder provider token"); + + assert!( + format!("{err:?}").contains("ec.identity_providers[example-provider].secrets.token"), + "should mention the provider token field" + ); + } + #[test] fn reject_placeholder_secrets_reports_both_partner_tokens() { let mut settings = diff --git a/crates/trusted-server-core/src/test_support.rs b/crates/trusted-server-core/src/test_support.rs index 37d54614c0..eae94aa1dd 100644 --- a/crates/trusted-server-core/src/test_support.rs +++ b/crates/trusted-server-core/src/test_support.rs @@ -7,6 +7,35 @@ pub mod tests { use crate::consent::types::{ConsentContext, ConsentSource, TcfConsent}; use crate::settings::Settings; + /// GDPR consent with TCF Purposes 1 and 4 and the given vendor consents. + #[must_use] + pub fn consent_granting_vendors(vendor_consents: Vec) -> ConsentContext { + let mut purposes = vec![false; 24]; + purposes[0] = true; + purposes[3] = true; + ConsentContext { + jurisdiction: Jurisdiction::Gdpr, + gdpr_applies: true, + tcf: Some(TcfConsent { + version: 2, + cmp_id: 1, + cmp_version: 1, + consent_screen: 0, + consent_language: "EN".to_owned(), + vendor_list_version: 1, + tcf_policy_version: 4, + created_ds: 0, + last_updated_ds: 0, + purpose_consents: purposes, + purpose_legitimate_interests: vec![false; 24], + vendor_consents, + vendor_legitimate_interests: Vec::new(), + special_feature_opt_ins: vec![false; 12], + }), + ..ConsentContext::default() + } + } + #[must_use] pub fn crate_test_settings_str() -> String { r#" @@ -605,3 +634,20 @@ pub mod nextjs_auction { #[cfg(feature = "test-utils")] pub(crate) mod async_html_finalization; + +/// Fixtures shared with adapter crates through the `test-utils` feature. +#[cfg(any(test, feature = "test-utils"))] +pub mod fixtures { + /// Returns an enabled identity provider configuration for tests. + /// + /// # Panics + /// + /// Panics if the embedded TOML is invalid. + #[must_use] + pub fn enabled_identity_provider() -> crate::settings::IdentityProviderConfig { + toml::from_str( + "id = \"example-provider\"\nkind = \"example-kind\"\nendpoint = \"https://ids.example.com/v1\"\nallowed_domains = [\"ids.example.com\"]\nsources = [\"ids.example.com\"]", + ) + .expect("should parse identity provider") + } +} diff --git a/crates/trusted-server-identity-providers/Cargo.toml b/crates/trusted-server-identity-providers/Cargo.toml new file mode 100644 index 0000000000..1ff50dedf8 --- /dev/null +++ b/crates/trusted-server-identity-providers/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "trusted-server-identity-providers" +description = "Identity provider implementations for Trusted Server." +authors = { workspace = true } +edition = { workspace = true } +license = { workspace = true } +publish = { workspace = true } +version = { workspace = true } + +[lib] +doctest = false + +[lints] +workspace = true + +[dependencies] +error-stack = { workspace = true } +trusted-server-core = { workspace = true } + +[dev-dependencies] +toml = { workspace = true } diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs new file mode 100644 index 0000000000..e11e69abf3 --- /dev/null +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -0,0 +1,254 @@ +//! Identity provider implementations for Trusted Server. +//! +//! Each provider kind is a module implementing +//! [`trusted_server_core::ec::provider::ProviderKind`]. [`build`] maps a +//! configured `kind` to its implementation; [`build_coordinator`] and +//! [`build_after_response_coordinator`] build the coordinators for enabled +//! providers collected before and after the response. +//! +//! Dependencies point one way: this crate depends on core, and core never +//! depends on it. + +use std::sync::Arc; +use std::time::Duration; + +use error_stack::Report; +use trusted_server_core::ec::coordinator::{ProviderCoordinator, RegisteredProvider}; +use trusted_server_core::ec::provider::{IdentityProvider, ProviderConfigError}; +use trusted_server_core::ec::registry::PartnerRegistry; +use trusted_server_core::error::TrustedServerError; +use trusted_server_core::settings::{ + CollectTiming, Ec, IdentityProviderConfig, ProviderSecrets, Settings, +}; + +/// Whether `kind` has a registered implementation. +/// +/// No kinds are registered yet. +#[must_use] +pub fn is_registered_kind(_kind: &str) -> bool { + false +} + +/// Builds the partner registry, marking the sources claimed by enabled +/// providers as provider-owned. +/// +/// # Errors +/// +/// Returns the same errors as [`PartnerRegistry::from_ec_with_kinds`], +/// including for an enabled provider whose kind is not registered. +pub fn partner_registry(ec: &Ec) -> Result> { + PartnerRegistry::from_ec_with_kinds(ec, is_registered_kind) +} + +/// Builds the provider for a configured `kind`. +/// +/// No kinds are registered yet, so every kind is rejected. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError::UnknownKind`] for an unregistered kind, or +/// any error from [`trusted_server_core::ec::provider::build_kind`]. +pub fn build( + config: &IdentityProviderConfig, + _secrets: &ProviderSecrets, +) -> Result, Report> { + let kind = config.kind(); + Err(Report::new(ProviderConfigError::UnknownKind) + .attach(format!("kind `{kind}` is not registered"))) +} + +/// Builds the coordinator for enabled providers collected before the +/// response (`collect = "page"`, the default). +/// +/// Returns `None` when no such provider is enabled. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError`] when a provider cannot be built or the +/// enabled providers, taken together, fail the coordinator's configuration +/// checks. +pub fn build_coordinator( + settings: &Settings, +) -> Result, Report> { + let partners = partner_registry(&settings.ec) + .map_err(|report| report.change_context(ProviderConfigError::Invalid))?; + build_coordinator_with(settings, Arc::new(partners)) +} + +/// Builds the page-phase coordinator against an already built partner +/// registry. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError`] when a provider cannot be built or fails +/// the coordinator's configuration checks. +pub fn build_coordinator_with( + settings: &Settings, + partners: Arc, +) -> Result, Report> { + build_phase_coordinator( + settings, + partners, + CollectTiming::Page, + Duration::from_millis(settings.ec.identity_budget_ms), + ) +} + +/// Builds the coordinator for enabled providers collected after the response +/// has been sent (`collect = "after_response"`), against an already built +/// partner registry. +/// +/// Its budget is `identity_after_response_budget_ms`. Returns `None` when no +/// such provider is enabled. +/// +/// # Errors +/// +/// Returns the same errors as [`build_coordinator_with`]. +pub fn build_after_response_coordinator( + settings: &Settings, + partners: Arc, +) -> Result, Report> { + build_phase_coordinator( + settings, + partners, + CollectTiming::AfterResponse, + Duration::from_millis(settings.ec.identity_after_response_budget_ms), + ) +} + +fn build_phase_coordinator( + settings: &Settings, + partners: Arc, + timing: CollectTiming, + budget: Duration, +) -> Result, Report> { + // Source ownership across both phases is checked when settings load + // (`config::validate_identity_providers`), so only this phase's providers + // are built and checked here. + let registered = settings + .ec + .identity_providers + .iter() + .filter(|config| config.enabled && config.collect == timing) + .map(|config| { + build(config, &config.secrets) + .map(|provider| RegisteredProvider::new(config.clone(), provider)) + }) + .collect::, Report>>()?; + if registered.is_empty() { + return Ok(None); + } + ProviderCoordinator::new(registered, partners, budget).map(Some) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn create_test_settings() -> Settings { + Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "example-admin-password" + + [publisher] + domain = "www.example.com" + cookie_domain = ".example.com" + origin_url = "https://origin.example.com" + proxy_secret = "example-proxy-secret" + + [ec] + passphrase = "example-passphrase-32-bytes-minimum" + "#, + ) + .expect("should parse test settings") + } + + #[test] + fn unknown_kind_is_rejected() { + let config: IdentityProviderConfig = toml::from_str( + "id = \"p\"\nkind = \"missing\"\nendpoint = \"https://ids.example.com/\"\nallowed_domains = [\"ids.example.com\"]", + ) + .expect("should parse"); + + let error = build(&config, &ProviderSecrets::default()) + .err() + .expect("should reject unknown kind"); + + assert!( + matches!(error.current_context(), ProviderConfigError::UnknownKind), + "should report unknown kind" + ); + } + + #[test] + fn no_enabled_provider_builds_no_coordinator() { + let mut settings = create_test_settings(); + settings.ec.identity_providers = vec![ + toml::from_str( + "id = \"p\"\nkind = \"missing\"\nenabled = false\nendpoint = \"https://ids.example.com/\"", + ) + .expect("should parse"), + ]; + + let coordinator = build_coordinator(&settings).expect("should skip disabled providers"); + + assert!(coordinator.is_none(), "should build no coordinator"); + } + + #[test] + fn enabled_unknown_kind_fails_the_build() { + let mut settings = create_test_settings(); + settings.ec.identity_providers = vec![ + toml::from_str( + "id = \"p\"\nkind = \"missing\"\nendpoint = \"https://ids.example.com/\"\nsources = [\"ids.example.com\"]", + ) + .expect("should parse"), + ]; + + let Err(error) = build_coordinator(&settings) else { + panic!("should fail instead of skipping an enabled provider of an unknown kind"); + }; + + let message = format!("{error:?}"); + assert!( + message.contains("`p`") && message.contains("missing"), + "should name the provider and its unregistered kind: {message}" + ); + } + + #[test] + fn rejects_an_unknown_collect_timing() { + toml::from_str::( + "id = \"example-provider\"\nkind = \"example-kind\"\ncollect = \"later\"", + ) + .expect_err("should reject an unknown `collect` value"); + } + + #[test] + fn parses_an_optional_hourly_rate_limit() { + let limited: IdentityProviderConfig = toml::from_str( + "id = \"example-provider\"\nkind = \"example-kind\"\nendpoint = \"https://ids.example.com/v1\"\nrate_limit_per_hour = 2", + ) + .expect("should parse a rate limit"); + let unset: IdentityProviderConfig = + toml::from_str("id = \"example-provider\"\nkind = \"example-kind\"\nendpoint = \"https://ids.example.com/v1\"") + .expect("should parse without a rate limit"); + + assert_eq!( + limited.rate_limit_per_hour.map(std::num::NonZeroU32::get), + Some(2), + "should keep the hourly limit" + ); + assert!( + unset.rate_limit_per_hour.is_none(), + "should leave the limit unset by default" + ); + toml::from_str::( + "id = \"example-provider\"\nkind = \"example-kind\"\nendpoint = \"https://ids.example.com/v1\"\nrate_limit_per_hour = 0", + ) + .expect_err("should reject a zero limit"); + } +} diff --git a/crates/trusted-server-integration-tests/tests/common/ec.rs b/crates/trusted-server-integration-tests/tests/common/ec.rs index 0a1f149c3b..1846fb9602 100644 --- a/crates/trusted-server-integration-tests/tests/common/ec.rs +++ b/crates/trusted-server-integration-tests/tests/common/ec.rs @@ -113,6 +113,7 @@ impl EcTestClient { .client .get(format!("{}{path}", self.base_url)) .header("sec-fetch-dest", "document") + .header("sec-fetch-mode", "navigate") .header("accept", "text/html"); for (key, value) in headers { builder = builder.header(*key, *value); diff --git a/docs/guide/api-reference.md b/docs/guide/api-reference.md index a84707f492..83d5806926 100644 --- a/docs/guide/api-reference.md +++ b/docs/guide/api-reference.md @@ -363,8 +363,10 @@ Server-to-server batch sync endpoint for writing EC ID to partner UID mappings. **Contract:** The JSON body is limited to 2 MiB and 1,000 mappings. The per-partner, per-minute limit is `ec.partners[].batch_rate_limit`. Responses are `200` when every mapping is accepted or `207` when at least one is rejected; -item reasons are `invalid_ec_id`, `invalid_partner_uid`, `ineligible`, or -`kv_unavailable`. Authentication failure returns `401 {"error":"invalid_token"}`; +item reasons are `invalid_ec_id`, `invalid_partner_uid`, `ineligible`, +`provider_owned` (the source belongs to a server-side identity provider), +`row_full` (the EC's row would exceed its 16 KiB limit; retrying gives the +same result), or `kv_unavailable`. Authentication failure returns `401 {"error":"invalid_token"}`; rate exhaustion returns `429`; an oversized body returns `413`; too many mappings returns `400`. The endpoint emits JSON but defines no dedicated cache or CORS headers. It is Fastly-only and requires the EC KV store. diff --git a/docs/guide/architecture.md b/docs/guide/architecture.md index fb02fe2e68..c4b2c5c2b4 100644 --- a/docs/guide/architecture.md +++ b/docs/guide/architecture.md @@ -172,7 +172,8 @@ Fastly and Axum support concurrent auction provider fan-out. Cloudflare and Spin currently accept at most one provider in an enabled auction. Every adapter runs target-aware fan-out and backend-name checks at startup. Auction collection uses an absolute cutoff for network waiting across headers and complete response bodies. -A nonblocking snapshot retains complete results after a slow origin. Fastly +A final snapshot retains complete results after a slow origin (on Fastly it may +wait up to 1 ms per partially received body). Fastly abandons pending backend requests because its SDK cannot cancel them; other adapters cancel their owned operations at cutoff. See [Auction timeout behavior](/guide/auction-orchestration#timeout-behavior) for diff --git a/docs/guide/auction-orchestration.md b/docs/guide/auction-orchestration.md index 2dd9d3016b..e83d7a2e1b 100644 --- a/docs/guide/auction-orchestration.md +++ b/docs/guide/auction-orchestration.md @@ -831,15 +831,15 @@ Until that callback lands, startup remains the mandatory target-aware gate. Auction requests start at dispatch and progress while the publisher origin loads. Response collection waits for headers and complete upstream bodies only until the -original absolute deadline. At expiry, one nonblocking snapshot preserves complete -responses already available; unfinished providers become timeouts. A slow origin +original absolute deadline. At expiry, one final snapshot preserves complete +responses already available; unfinished providers become timeouts. On Fastly the snapshot may wait up to 1 ms per partially received body so the host can surface bytes that already arrived, and reads on into the body of a response whose headers arrived by the cutoff instead of abandoning it. A slow origin never restarts the budget or causes an already-buffered bid to be discarded. Fastly has no upstream arrival timestamp, so this is a bound on network waiting, not bid eligibility based on an exact server-side arrival time. A mediator starts only with remaining budget and uses the earlier of the auction cutoff and its own logical timeout. -Provider bids that are already complete at the final nonblocking snapshot remain +Provider bids that are already complete at the final snapshot remain eligible even after `tmax`; mediation starts only with remaining budget and rejects responses collected after its cutoff. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index d60fb7f810..29bdb7d9d4 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -60,6 +60,7 @@ publisher, trusted-client-IP, EC, handler, Tinybird, DataDome, and S3 fields: - `ec.passphrase` - `ec.partners[*].api_token`, when inbound identify or batch sync is used - `ec.partners[*].ts_pull_token`, when pull sync is enabled +- `ec.identity_providers[*].secrets.token`, when a provider needs an API token - `handlers[*].password` - `tinybird.auction_token_secret`, when Tinybird auction telemetry is enabled - `integrations.datadome.server_side_key_secret_name`, when protection is enabled @@ -858,14 +859,17 @@ Legacy consent-store records are not read or migrated into `ec.ec_store`. Their `passphrase` is a key name in `trusted_server_secrets`; the resolved value must be at least 32 bytes. Keep it stable to preserve EC identifier continuity. -| Field | Type | Required | Description | -| ------------------------- | -------------- | -------- | ----------------------------------------------------------------------- | -| `passphrase` | String | Yes | Publisher passphrase used as HMAC key | -| `ec_store` | String or null | No | Fastly KV store name for EC identity graph and withdrawal state | -| `pull_sync_concurrency` | Integer | No | Maximum concurrent pull-sync requests per organic response | -| `cluster_trust_threshold` | Integer | No | Cluster size threshold for identity trust decisions | -| `cluster_recheck_secs` | Integer | No | Legacy compatibility setting; cluster rechecks no longer use timestamps | -| `partners` | Array | No | Static partner registry entries | +| Field | Type | Required | Description | +| ----------------------------------- | -------------- | -------- | ----------------------------------------------------------------------- | +| `passphrase` | String | Yes | Publisher passphrase used as HMAC key | +| `ec_store` | String or null | No | Fastly KV store name for EC identity graph and withdrawal state | +| `pull_sync_concurrency` | Integer | No | Maximum concurrent pull-sync requests per organic response | +| `cluster_trust_threshold` | Integer | No | Cluster size threshold for identity trust decisions | +| `cluster_recheck_secs` | Integer | No | Legacy compatibility setting; cluster rechecks no longer use timestamps | +| `partners` | Array | No | Static partner registry entries | +| `identity_budget_ms` | Integer | No | Shared identity lookup budget, 1–1000 ms (default 200) | +| `identity_after_response_budget_ms` | Integer | No | After-response lookup budget, 1–5000 ms (default 2000) | +| `identity_providers` | Array | No | Server-side identity providers (see below) | ::: tip Partner keying `source_domain` is the canonical partner key. It matches incoming OpenRTB EID `source` values and is also used as the EC KV `ids` map key. @@ -898,6 +902,119 @@ TRUSTED_SERVER__EC__PASSPHRASE=ec_passphrase TRUSTED_SERVER__EC__EC_STORE=ec_identity_store ``` +### Identity providers + +`[[ec.identity_providers]]` entries look up a visitor's IDs from a provider's +server on publisher page navigations. Lookups start before the origin fetch, +share the `identity_budget_ms` budget, and are collected before the response is +finalized. Results go into the navigation's single EC row write. A lookup that +times out or fails leaves the row unchanged. + +Lookups run only for an EC whose row already exists (an EC minted on this +navigation is looked up on the next one), only when consent allows EID +persistence, and, when `tcf_vendor_id` is set and TCF applies to the request, +only with TCF vendor consent for that vendor. The same vendor consent gates +egress of the provider's `sources`, even while the provider is disabled: while +it is withheld, every ID for those sources, whether a provider stored it or the +browser sent it in `ts-eids` or the `/auction` body, is left out of auction +`user.eids`, `x-ts-eids` and `/_ts/api/v1/identify`. Removing the provider's +entry removes that gate, so keep a provider with `enabled = false` while its +stored UIDs may still be live. Without `tcf_vendor_id`, a provider is gated by +the EID-persistence purposes alone, so set it for any provider that has a TCF +vendor ID. + +Identity providers run only on Fastly. This release registers no provider kind, +so `ts config validate`, `push` and `diff` reject an enabled provider; keep +`enabled = false` until a kind ships. The Axum, Cloudflare and Spin adapters +refuse to start when an enabled provider is configured. On Fastly, an enabled +provider whose `kind` this build does not include, or whose `settings` its kind +rejects, also fails startup. Lookups always verify the provider's TLS +certificate, whatever `proxy.certificate_check` says. + +A provider owns its `sources`: its results replace whatever the row holds for +them, and batch sync rejects them and pull sync skips them. Browser-resolved IDs +(`ts-eids`, `sharedId`) are never stored for an owned source, so a source a +provider cleared on an opt-out is not refilled from the browser. + +When a provider reports an opt-out, its sources are cleared and the row +remembers the opt-out for 30 days. The provider is not called for that EC +again until the 30 days pass. + +| Field | Type | Required | Description | +| --------------------- | ------- | -------- | ----------------------------------------------------------------------- | +| `id` | String | Yes | Stable provider ID used in logs, metrics and KV ownership | +| `kind` | String | Yes | Provider implementation; must be one this build includes | +| `enabled` | Boolean | No | Whether the provider runs (default `true`) | +| `endpoint` | String | Yes | HTTPS lookup URL; its host must be in `allowed_domains` | +| `allowed_domains` | Array | Yes | Hosts the provider may call | +| `sources` | Array | Yes | Source domains the provider owns; each needs an `[[ec.partners]]` entry | +| `tcf_vendor_id` | Integer | No | TCF vendor ID that must have vendor consent | +| `collect` | String | No | `"page"` (default) or `"after_response"`; see below | +| `rate_limit_per_hour` | Integer | No | Most lookups per EC hash per hour; unset means unlimited. See below | +| `secrets.token` | String | No | Key in `trusted_server_secrets` for the provider's API token | +| `settings` | Table | No | Provider-specific settings, parsed by the provider kind | + +**Example**: + +```toml +[ec] +identity_budget_ms = 200 + +[[ec.partners]] +name = "Example IDs" +source_domain = "ids.example.com" + +[[ec.identity_providers]] +id = "example-provider" +kind = "example-kind" +# This release registers no provider kind; an enabled provider fails deploy +# validation and startup. +enabled = false +endpoint = "https://ids.example.com/v1/lookup" +allowed_domains = ["ids.example.com"] +sources = ["ids.example.com"] + +[ec.identity_providers.secrets] +token = "example_provider_token" +``` + +`collect = "after_response"` lookups run after the response is sent, within +`identity_after_response_budget_ms`, and only when `ec_store` is configured, +the response is not a redirect, the navigation's own write succeeded (or had +nothing to change), the row is live, and the EC was not minted on this +navigation; a row recovered from an orphaned cookie gets a newly minted EC, so +its lookups wait for the next navigation. They also run behind an error +response, including a `502` that Trusted Server generates when the origin is +unreachable. Their results reach the row in a follow-up write. Providers run +only on Fastly. + +::: warning Rate limit granularity +`rate_limit_per_hour` is counted per EC hash, which is derived from the client +IP, so ECs behind one IP (a NAT or carrier gateway) share the budget. On +Fastly it is enforced as a per-minute budget rounded up to at least one lookup +per minute, so any value below 60 behaves like 60 per hour. Values of 60 and +above are enforced as `ceil(value / 60)` per minute. +::: + +**Validation**: configuration validation fails when an enabled provider's +`endpoint` is not HTTPS or its host is not in `allowed_domains`, when it claims +no sources, when a source has no `[[ec.partners]]` entry, when two enabled +providers claim the same source, or when its `id` is empty or shared with +another enabled provider. `identity_budget_ms` must be 1–1000 and +`identity_after_response_budget_ms` 1–5000. Deploy validation also rejects an +empty `secrets.token` key name, and, because this release registers no +provider kind, any enabled provider. A disabled provider's `secrets.token` is +not resolved. + +::: warning Rollback +A config blob that configures identity providers, or sets either identity +budget to a non-default value, cannot be loaded by a build from before +identity providers. Roll the runtime back only with a blob that leaves them +out. Rows holding a provider UID over 512 bytes also fail to load on such a +build, and an older writer drops a row's `eids` and provider opt-outs when it +rewrites the row. +::: + ### Field Details #### `passphrase` @@ -2560,11 +2677,11 @@ push as proof that a Cloudflare or Spin multi-provider plan can start. ### Deadline behavior Auction response collection waits only until an absolute deadline measured from -dispatch, including headers and complete response bodies. One nonblocking snapshot +dispatch, including headers and complete response bodies. One final snapshot retains complete available responses after a slow origin; unfinished providers become timeouts and mediation cannot start with an exhausted budget. Backend -timeout rounding does not extend network waiting. -Provider bids that are already complete at the final nonblocking snapshot remain +timeout rounding does not extend network waiting. On Fastly the snapshot may wait up to 1 ms per partially received body so the host can surface bytes that already arrived, and reads on into the body of a response whose headers arrived by the cutoff instead of abandoning it. +Provider bids that are already complete at the final snapshot remain eligible even after `tmax`; mediation starts only with remaining budget and rejects responses collected after its cutoff. diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index 113066cdaf..c137d6ddbe 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -42,7 +42,7 @@ maps to a stable base. ### Request Lifecycle -Every request passes through four phases. EC generation only happens on organic routes (publisher proxy, integration proxy, auction) — read-only endpoints like `/identify` and `/batch-sync` skip generation entirely. During pre-routing, Trusted Server builds consent from request-local cookies, headers, geolocation, and policy defaults; it does not load consent from a separate KV store. +Every request passes through four phases. A new EC is minted, and its row created, only on a top-level publisher navigation: a `GET` through the publisher proxy with `Sec-Fetch-Dest: document`, `Sec-Fetch-Mode: navigate` and no `prefetch` token in `Sec-Purpose`, answered with a response that is not a 301, 302, 303, 307 or 308 redirect. Clients that send no Fetch Metadata get no EC, and a prefetch or prerender (`Sec-Purpose: prefetch;prerender`) never mints or writes. Integration proxies and `/auction` use an existing EC but never mint one, and read-only endpoints like `/identify` and `/batch-sync` skip generation entirely. During pre-routing, Trusted Server builds consent from request-local cookies, headers, geolocation, and policy defaults; it does not load consent from a separate KV store. ```mermaid sequenceDiagram @@ -78,7 +78,7 @@ flowchart TD ConsentCheck -- "No" --> ExplicitWithdrawal{Explicit
withdrawal?} ExplicitWithdrawal -- "Yes" --> CookiePresent{Cookie was
present?} - CookiePresent -- "Yes" --> Withdraw["Expire ts-ec cookie
Write withdrawal tombstone in ec_identity_store (24h TTL)
Strip all x-ts-* headers"] + CookiePresent -- "Yes" --> Withdraw["Write withdrawal tombstone in ec_identity_store (24h TTL)
Expire ts-ec cookie once it lands
Strip all x-ts-* headers"] CookiePresent -- "No" --> HeaderOnly["Strip all x-ts-* headers only
(no cookie expiry or KV tombstone)"] ExplicitWithdrawal -- "No" --> HeaderOnly @@ -133,7 +133,7 @@ Partner identities flow into the KV identity graph through three channels. Each ```mermaid flowchart LR subgraph Browser-initiated - Prebid["Prebid EIDs
/auction body + ts-eids + sharedId
Response-time ingestion"] + Prebid["Prebid EIDs
ts-eids + sharedId
Navigation-time ingestion"] end subgraph Server-initiated @@ -148,13 +148,12 @@ flowchart LR ### Prebid EID Flow -Browser-resolved Prebid EIDs reach the identity graph when Trusted Server finalizes a response. It collects matched partner UIDs from three request-local sources, applied in this order so that, when two sources carry the same partner, the later source's UID is the candidate offered to KV: +Browser-resolved Prebid EIDs reach the identity graph when Trusted Server finalizes a top-level document navigation, the only request that writes browser-resolved IDs to KV. It collects matched partner UIDs from two request-local sources, applied in this order so that, when both carry the same partner, the later source's UID is the candidate offered to KV: 1. the `ts-eids` cookie, -2. the `eids` array in the `/auction` request body, -3. the `sharedId` cookie. +2. the `sharedId` cookie. -The `/auction` body carries the full EID set Prebid.js resolved, so `/auction` ingestion is not limited by the size cap TSJS applies to the `ts-eids` cookie. The cookie still bridges Prebid user ID modules with the identity graph on requests that carry no EID body, such as `GET /_ts/page-bids` and page navigations. +The `eids` array in the `/auction` request body goes to bidders on that request only and is never stored. The `ts-eids` cookie bridges Prebid user ID modules with the identity graph on the next navigation. These Prebid EID writes follow the same consent rule as bidstream EID forwarding: when a TCF signal is present, Purpose 1 (store/access device) and Purpose 4 (personalized ads) must both be consented, and when GDPR applies without a TCF signal the writes are withheld. A user who consents to Purpose 1 but not Purpose 4 keeps their EC, but no EIDs from the request are persisted. @@ -173,8 +172,8 @@ sequenceDiagram TSJS->>TSJS: Base64 encode full OpenRTB-style EID array
[{source, uids:[{id, atype, ext?}]}] TSJS->>B: document.cookie = "ts-eids=..." - Note over B,TS: Next eligible EID-sync request - B->>TS: Document navigation, POST /auction,
or admitted GET /_ts/page-bids + Note over B,TS: Next page load + B->>TS: Top-level document navigation TS->>TS: Base64 decode → parse OpenRTB-style EIDs
match source domains to partners TS->>KV: Add missing partner IDs in one conditional write
skip when UIDs already match
(requires TCF Purpose 1 + 4 under GDPR) ``` @@ -183,11 +182,11 @@ Current TSJS writers preserve the full OpenRTB-style `{source, uids:[...]}` shap The `sharedId` cookie follows a similar path but is written directly by Prebid's SharedID module rather than by TSJS. The server reads it separately and maps it via the `sharedid.org` source domain. -Returning-user cookie persistence runs only on publisher document navigations, `POST /auction`, and admitted `GET /_ts/page-bids` SPA navigations. Static assets, analytics, integration requests, filter short circuits, and other subresources do not decode or persist these cookies. New EC creation remains eligible regardless of route because its backing row must include the request's initial IDs. +Cookie persistence runs only on top-level publisher navigations: a `GET` through the publisher proxy with `Sec-Fetch-Dest: document`, `Sec-Fetch-Mode: navigate` and no `prefetch` in `Sec-Purpose`. `POST /auction`, `GET /_ts/page-bids`, static assets, analytics, integration requests, filter short circuits, and other subresources do not persist these cookies. -Each eligible request attempts at most one conditional cookie update. If another writer wins, Trusted Server reads the row once and does not write again from that request. A matching value completes the sync; an absent or different value is deferred. +Each navigation makes at most one conditional page-load write, which carries browser ingest and page-phase identity-provider results together. If another writer wins, the navigation's changes are dropped without a re-read; a later navigation offers them again. Two writes sit outside that rule: a consent withdrawal's tombstone is written on any request that carries the EC cookie, with bounded retries, and `collect = "after_response"` identity providers make one follow-up write after the response is sent. Batch sync and pull sync write through their own channels. -Browser cookies and the `/auction` request body do not carry a trustworthy value timestamp or sequence. Trusted Server therefore adds missing partner IDs but does not replace a different stored UID from a browser-supplied EID, even on later eligible requests. Pull- or push-enabled partners can replace that value through their authoritative synchronization path. A cookie-only partner retains the stored UID until an authoritative freshness rule is introduced or the EC row expires. +Browser cookies do not carry a trustworthy value timestamp or sequence. Trusted Server therefore adds missing partner IDs but does not replace a different stored UID from a browser-supplied EID, even on later eligible requests. Pull- or push-enabled partners can replace that value through their authoritative synchronization path. A cookie-only partner retains the stored UID until an authoritative freshness rule is introduced or the EC row expires. ### EID Seeding and Prebid Bidstream Forwarding @@ -263,7 +262,7 @@ The relevant OpenRTB structure forwarded to Prebid Server and downstream partner } ``` -Server-resolved EIDs and current-request Prebid EIDs are deduplicated by `source + uid.id`. Browser-supplied EIDs, from the `/auction` request body, the `ts-eids` cookie, or the `sharedId` cookie, only fill partner UIDs that are missing from KV; a different stored UID is kept, as described in [Prebid EID Flow](#prebid-eid-flow). +Server-resolved EIDs and current-request Prebid EIDs are deduplicated by `source + uid.id`. Browser-supplied EIDs from the `ts-eids` or `sharedId` cookie only fill partner UIDs that are missing from KV (the `/auction` request body goes to bidders only and is never stored); a different stored UID is kept, as described in [Prebid EID Flow](#prebid-eid-flow). ### Pull-Sync Completeness Marker @@ -271,7 +270,7 @@ When the identity graph contains a UID for every pull-enabled partner, Trusted S A valid marker avoids a KV lookup only when pull-sync completeness is the sole reason to inspect the row. Auctions that need stored EIDs, browser EID-cookie ingestion, explicit withdrawal, and generation continue to use KV. The marker acts as recent proof that the row existed, so deletion of a previously complete row is not detected until the marker expires, at most one hour after issuance. Partner-set changes, passphrase rotation, malformed values, and expiration invalidate the marker and restore the normal lookup and orphan-recovery path. -Pull sync runs after response delivery, so a partner response that fills the last missing UID cannot set the marker on that already-sent response. A later eligible request verifies the completed row and issues the marker. Explicit withdrawal expires both `ts-ec` and any present `ts-ec-pull-complete` marker even when KV is unavailable. +Pull sync runs after response delivery, so a partner response that fills the last missing UID cannot set the marker on that already-sent response. A later eligible request verifies the completed row and issues the marker. Explicit withdrawal always expires any present `ts-ec-pull-complete` marker, and expires `ts-ec` once the row is tombstoned or proven absent. While the tombstone cannot be written (KV unavailable, CAS retries exhausted, or a row that cannot be read), `ts-ec` is kept so the next request that carries it retries the withdrawal. Issuing or expiring the marker adds `Set-Cookie` to the outgoing response. Cache-privacy handling makes an otherwise shareable response private when that happens. A valid marker is not refreshed on each request, so this cost is limited to responses that establish or clear marker state in exchange for avoiding later KV reads. diff --git a/docs/guide/integrations/prebid.md b/docs/guide/integrations/prebid.md index b7a05ac7be..2de92cddcf 100644 --- a/docs/guide/integrations/prebid.md +++ b/docs/guide/integrations/prebid.md @@ -865,12 +865,10 @@ page or auction. When available, the opaque value follows the standard path: 2. The current `/auction` request includes that entry. 3. Trusted Server merges and consent-gates it, then forwards it to Prebid Server as `user.ext.eids`. -4. After the response, the same `/auction` request ingests the body entry into - an EC/KV partner configured with `source_domain = "liveramp.com"`, when that - partner UID is not already stored. -5. The browser also persists the opaque value in the bounded `ts-eids` cookie, - which lets requests without an EID body, such as `GET /_ts/page-bids` and - page navigations, ingest it. +4. The browser also persists the opaque value in the bounded `ts-eids` cookie. + The next top-level page navigation ingests it into an EC/KV partner + configured with `source_domain = "liveramp.com"`, when that partner UID is + not already stored. `/auction` and `GET /_ts/page-bids` never write KV. Trusted Server treats the RampID envelope as an opaque string. Do not log, decode, publish, or dimension metrics by the value. Source names, counts, @@ -938,7 +936,7 @@ domain, booleans, source names, counts, and status codes: recording its value. 5. Confirm a controlled Prebid Server request contains that source in `user.ext.eids`. -6. Confirm a later request ingests the source into the configured +6. Confirm a later top-level navigation ingests the source into the configured `liveramp.com` EC partner. 7. Repeat with denied consent and confirm the envelope endpoint is not called, `idl_env` is not written, and no LiveRamp EID is forwarded. @@ -958,9 +956,9 @@ Trusted Server uses a **hybrid EID forwarding model** for Prebid-routed auctions 2. **Server-side EIDs from the EC/KV identity graph** are resolved on the edge from the current EC ID. 3. Trusted Server **merges and deduplicates** both sets before calling Prebid Server. 4. The merged result is forwarded downstream as `user.ext.eids` in the OpenRTB request. -5. After the response, Trusted Server adds matched partner UIDs that are missing from the EC identity graph, reading the `ts-eids` cookie, then the `/auction` request-body EIDs, then the `sharedId` cookie. Because the body carries the untrimmed EID set, `/auction` ingestion is not limited by the size-capped `ts-eids` cookie. +5. On the next top-level page navigation, Trusted Server adds matched partner UIDs that are missing from the EC identity graph, reading the `ts-eids` cookie, then the `sharedId` cookie. The `/auction` request-body EIDs go to bidders on that request only and are never stored, so ingestion is limited by the size-capped `ts-eids` cookie. -The `ts-eids` cookie still matters for requests without an EID body, such as `GET /_ts/page-bids` and page navigations, where it is the source for both EID fallback and identity-graph ingestion. Identity-graph EID writes follow the same consent rule as bidstream forwarding: under GDPR, TCF Purpose 1 and Purpose 4 must both be consented. +The `ts-eids` cookie also supplies the EID fallback for requests without an EID body, such as `GET /_ts/page-bids`. Identity-graph EID writes follow the same consent rule as bidstream forwarding: under GDPR, TCF Purpose 1 and Purpose 4 must both be consented. This means Prebid auctions get same-request transparency for browser-resolved IDs without giving up the durability of the server-managed EC identity graph. @@ -982,7 +980,7 @@ sequenceDiagram T->>P: OpenRTB request\nuser.ext.eids = merged set P-->>T: OpenRTB bid response T-->>B: Auction response - T->>K: Ingest body eids + ts-eids/sharedId cookies\n(TCF Purpose 1 + 4 under GDPR) + Note over B,K: /auction writes nothing to KV; ts-eids and sharedId\nare ingested on the next top-level navigation ``` ### Merge and deduplication rules diff --git a/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md b/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md new file mode 100644 index 0000000000..a955b2b463 --- /dev/null +++ b/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md @@ -0,0 +1,1784 @@ +# Server-to-Server EID Collection — Design Spec + +**Date:** 2026-10-30 +**Status:** PR A (#1487) implements PR1 to PR4, PR B (#1489) PR5 and PR6; +reviewed (see the review log) +**Source of intent:** the S2S EID collection requirements document, kept +outside this repository (sections referenced as "R§n") + +## 1. Purpose + +Trusted Server (TS) collects external IDs (EIDs) only through the browser and +one fixed pull-sync protocol, and a single page load can write the EC KV row +several times. This spec defines: + +- a Rust trait that any identity provider implements so TS can collect IDs + server-to-server, with implementations isolated in a dedicated crate; +- one shared coordinator that owns eligibility, consent, transport, time + budget, validation, merging, persistence and the mirror cookie; +- a strict rule of **at most one KV write per page load**, made on the top-level + document navigation and never retried; an opt-in after-response phase + (§5.2.1) may add one guarded follow-up update with provider results only; +- a full-fidelity, ownership- and expiry-aware KV representation for + provider-written IDs, and how auctions and other consumers read it. + +Client-side collection (`ts-eids`, the `/auction` body) keeps working alongside +server-side collection, and both feed the same consent-gated KV row. + +## 2. Scope + +### 2.1 In scope + +- `IdentityProvider` trait, its input/output types and capability description. +- Dedicated provider crate boundary and registration. +- Identity coordinator: provider selection, consent gating, allowlisted + outbound HTTP, shared time budget, validation, merge, single write. +- Page-load write permit: navigation-only, single-attempt KV writes. +- Additive KV schema for multiple EIDs/UIDs with EID-level and UID-level + fields, owner and expiry. +- `__Host-ts-eids-mirror` first-party cookie. +- Consumer changes: `/auction`, `/_ts/page-bids`, `x-ts-eids`, + `/_ts/api/v1/identify`, browser ingest, batch sync on provider-owned sources. +- Re-expressing the existing pull sync as the first provider on the trait. +- Operator configuration, secrets, logs and metrics. + +### 2.2 Non-goals (firm, from R§6) + +- Logged-in identity and email contribution: no hashed-email intake, no + `ts-hem` cookie, no `tsjs.setEmail()`, no publisher sign-in changes. The + trait carries a typed `HashedEmail` slot that is always empty in V1; a source + that requires it is skipped before any call. TS never receives a raw email + from the page. +- Proxying vendor identity JavaScript through TS. +- Concrete vendor implementations (each gets its own spec, see §13.2 F1). +- Adapters other than Fastly. KV identity remains Fastly-only; Axum, + Cloudflare and Spin get this design when EC identity is ported to them. +- Cross-browser linking. +- Vendor segment products and ad-server targeting. +- Batch sync and admin write paths, except how batch sync treats a + provider-owned source (§8.5). +- Removing `ts-eids` or `x-ts-eids`. +- Storing the browser's complete EID set. Browser ingest keeps today's rules + (§8.4); full fidelity applies only to provider-written IDs. +- Unregistered sources for providers. A provider claims only sources that have + an `[[ec.partners]]` entry. + +### 2.3 Future tasks (R§5, not in this scope) + +Listed in §13.2 in their original order, stacked on `main`, each a Tech Lead +decision. + +## 3. Agreed decisions + +These were agreed before drafting and are binding on the rest of the spec. + +| # | Topic | Decision | +| --- | ------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| D1 | Which request may write | Only the top-level document navigation. `/auction`, `/_ts/page-bids` and integration-proxy requests never write KV. | +| D2 | Provider timing | Provider requests start when the navigation starts, run concurrently with the origin fetch, and are collected within a shared total budget of about 200 ms. Their results join the navigation's write. | +| D3 | Which navigations look up | Any navigation where an enabled provider has a due source (missing or expired UID). No-match results are not cached in V1. | +| D4 | Failed write | Not retried in that page load. KV and the mirror cookie stay unchanged. A new EC is not issued if its row was not created. The next navigation is a new page load and may write. | +| D5 | Availability | Provider IDs are guaranteed to `/auction` from the next page load. On the same page they are used if the write already completed. | +| D6 | Mirror cookie | JavaScript-readable. Holds provider-owned UIDs of `bidstream_enabled` sources only. Never trusted by the server over KV. Filled deterministically up to its size limit. | +| D6b | Mirror consumer | The tsjs change that reads the mirror and hands IDs to Prebid is a separate optional PR. | +| D7 | Expiry | Declared by the writing provider per UID. A UID without `exp` is served until replaced or removed. Browser and batch IDs carry no `exp` and no owner. | +| D8 | Conflicts | On a provider-owned source, the provider's UIDs replace that source's UIDs. Browser ingest fills only missing sources and never overwrites provider-owned UIDs. Batch sync rejects provider-owned sources. | +| D9 | Removal | Only consent withdrawal (the existing whole-row tombstone) is in scope. A tombstone wins over concurrent and later collection; IDs return only after consent is granted again. | +| D10 | Pull sync | Re-expressed as the first provider on the trait; the separate post-send dispatcher is removed. | +| D11 | Consent | Existing EID-persistence consent check, plus an optional per-provider TCF vendor ID that must also have vendor consent. | +| D12 | KV schema | Stay on `v: 1` with additive optional fields; no migration. | +| D13 | Other consumers | Provider IDs appear in `x-ts-eids` and `/_ts/api/v1/identify`, subject to `bidstream_enabled`, expiry and consent. | + +§5.2.1 (opt-in `collect = "after_response"`) and §5.6 (refresh state) extend +D2 and D3 additively; defaults are unchanged. + +### 3.1 Choices this spec makes to apply those decisions + +Reviewers should check these; each follows from a decision above. + +- **Navigation test (D1).** A request is a page-load navigation when it is a + `GET` on the publisher-proxy route with `Sec-Fetch-Dest: document` and + `Sec-Fetch-Mode: navigate`, and its `Sec-Purpose` header, if any, has no + `prefetch` token. Requests without Fetch Metadata never write. A navigation + whose final response is a 301, 302, 303, 307 or 308 redirect does not write; + the redirect target is the page load. A `304` keeps the permit. This is stricter than the existing `is_navigation_request` + (which falls back to `Accept: text/html`); that helper keeps its other uses, + and the write permit uses the stricter test. +- **Integration-proxy EC minting stops (D1).** Integration-proxy requests can no + longer create a KV row, so they no longer mint a new EC. Today a new EC is + only issued when its row exists, so this keeps that invariant. +- **Withdrawal is the one exception to the one-write rule (D1, D9).** Any + finalized request that carries a consent withdrawal and the EC cookie writes + the tombstone, navigation or not, keeping today's behaviour. It tombstones the + cookie's EC ID and the active EC ID (when they differ), using a bounded CAS + retry and, when a store listing proves the key exists, an unconditional + tombstone. It serves no IDs and expires the mirror cookie, and it expires the + EC cookie only when every tombstone succeeded, so a failed write is retried on + the next request instead of orphaning the row. Deferring the tombstone to the + next navigation would leave a withdrawn visitor's row live for as long as they + only make non-navigation requests, which privacy does not allow. +- **One owner per source (D8).** Config validation rejects two enabled providers + claiming the same source, so "the owning provider" of a source is always + unambiguous. +- **No retry loops on the navigation write (D4).** Existing loops on this path + (EC-ID collision on create, orphan recovery) become a single attempt. + +## 4. Terms + +| Term | Meaning | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| page load | One top-level document navigation and every request the page makes because of it: `/auction`, `/_ts/page-bids` (SPA view changes), integration proxies. | +| navigation | The request that starts a page load (§3.1). | +| row | The EC KV identity-graph entry keyed by EC ID ([`KvEntry`]). | +| source | An EID source domain registered in `[[ec.partners]]`. | +| provider | A configured `IdentityProvider` instance with an operator-chosen `ProviderId`. | +| owner | The provider that wrote a UID. Browser and batch UIDs have no owner. | +| provider-owned | A source claimed by an enabled provider. A source whose owner is disabled is not provider-owned; its owned UIDs serve until they expire or are pruned. | +| due source | A source an enabled provider claims whose owned UIDs are missing or all expired, or one of whose live owned UIDs has refresh state with `from <= now` (§5.6). | +| live UID | A stored UID with no `exp`, or with `exp` greater than now. | +| mirror | The `__Host-ts-eids-mirror` cookie. | + +## 5. End-to-end flow + +### 5.1 Inputs + +| Input | Arrives on | Persisted? | +| --------------------------------- | ----------------------------------------- | ------------------------------------------- | +| `ts-eids` cookie | every request | Navigation only, via browser ingest (§8.4) | +| `sharedId` cookie | every request | Navigation only, applied last in ingest | +| `/auction` body `user.eids` | `/auction` | Never; forwarded to bidders on that request | +| Provider lookup responses | navigation (outbound) | Navigation write | +| After-response provider responses | navigation, after the response (outbound) | Follow-up CAS update (§5.2.1) | +| Batch sync | its own endpoint | Its own write path (unchanged) | + +### 5.2 Navigation (the only writer) + +``` +browser TS (navigation) origin providers KV + | GET / (document) | | | | + |------------------>| read EC cookie, consent | | | + | | read row snapshot ---------------------------------->| + | | classify: navigation => PageLoadWrite permit | + | | coordinator.dispatch(): due sources, consent, | + | | allowlist ---------------------------> lookups | + | | fetch origin ------------------------>| (concurrent) | + | |<-------------------- origin headers --| | | + | | coordinator.collect(): wait until all done or | + | | dispatch + budget (≈200 ms); drop late responses | + | | validate + merge: browser ingest + provider results | + | | if anything changed: one write (permit consumed) ---->| + | |<---------------------------------------- ok / failed | + | | on ok: Set-Cookie EC (new rows) | +| | mirror from committed row, else snapshot row | + |<------------------| response streams | +``` + +Notes: + +- The budget is measured from dispatch. If the origin takes longer than the + budget, provider collection adds no latency. +- The added delay never exceeds the budget. Collection uses + `PlatformHttpClient::select_until` (added by PR #1433), which bounds both + header waits and body reads by an absolute deadline on every adapter, so a + slow or byte-trickling provider cannot hold the response. +- Response headers, including cookies, depend on the write result, so the write + completes before headers are sent. This is today's behaviour for new ECs. +- A navigation with nothing due and nothing new from the browser makes no + provider calls and no write. + +### 5.2.1 After-response collection (opt-in) + +A provider can be slower than any acceptable page budget (for example, one +that takes about a second). Setting `collect = "after_response"` on that +provider moves its lookup out of the page path: + +``` +browser TS (navigation) providers KV + | | page phase as in §5.2, without after-response providers + |<------------------| response sent (send_to_client) | | + | | dispatch after-response providers --------->| | + | | collect until after-response budget (≈2 s) | | + | | read row + generation --------------------------->| + | | one CAS update with their results (if changed) -->| +``` + +- Default is `collect = "page"`; with no provider set to `after_response`, + behaviour is exactly §5.2. No existing behaviour changes. +- Dispatch happens only after the response is sent, because `select_until` + abandons requests still pending at the page deadline on every adapter. +- The budget is `ec.identity_after_response_budget_ms` (default 2000), + measured from the after-response dispatch. Late responses are dropped as in + §8.1.1. +- Only a navigation whose permit is confirmed (not a redirect) and whose page + phase succeeded (consent allows the EC; the page write landed or had nothing + to change) runs the phase, with an `AfterResponseWrite` permit (§9.1). + Redirects and non-navigation requests get neither permit. +- The follow-up write contains provider results only; browser ingest stays in + the page write. It never creates or revives a row: a missing, tombstoned or + withdrawn row, or one whose EC ID changed during the request, is skipped. +- When no result can change the row (nothing found, nothing to clear), the + phase skips the read and the write and logs `unchanged`. A remembered + opt-out clears only sources the row still stores. +- The write is a compare-and-swap on the generation read just before it. On + conflict the results are dropped; the source stays due and is retried on the + next navigation. +- Results are not mirrored in the already-sent response. They are served from + KV from the next page load. A browser fallback UID for an owned source is + replaced by the provider's UID, as in §8.3. +- Source ownership is validated across both phases: a source claimed by a + page provider and an after-response provider is rejected at startup. +- Cost: the instance stays busy after the response for up to the + after-response budget, which lowers per-instance throughput when sandboxes + are reused. Keep the budget no larger than the slowest provider needs. +- Only the Fastly adapter runs identity providers (§2.2). Other adapters + refuse to start with an enabled provider. +- Cookie reads and generated IDs of an after-response provider also land only + in the follow-up write, so cookie-only providers should stay `page`. + +### 5.3 What one navigation write contains + +Exactly one of these, chosen from the request snapshot: + +| Situation | Write | +| --------------------------------------- | ----------------------------------------------------------------------------- | +| Consent withdrawn, live row | No page-load write; the tombstone follows §3.1. No lookups, no ingest. | +| Row tombstoned, no fresh consent | None. No lookups. Nothing served. | +| New EC (or orphaned EC being recovered) | Create the row with browser ingest. No lookups: the EC has no row yet (§8.1). | +| Returning EC, something changed | CAS update with browser ingest + valid provider results; expired UIDs pruned. | +| Returning EC, nothing changed | None. | + +### 5.4 Later requests in the same page load + +`/auction`, `/_ts/page-bids` and integration proxies read the row snapshot and +never write. They do not receive a permit, so the write API cannot be called +(§9.1). EIDs in the `/auction` body go to bidders on that request only. + +### 5.5 Lifecycle per ID origin + +| ID origin | Collected | Persisted | Mirrored | Served from KV to `/auction` | +| --------------------- | ------------------------------ | ----------------------------- | ---------------------- | --------------------------------------- | +| `ts-eids`/`sharedId` | Navigation carrying cookies | That navigation's write | Never | Next page load (same page via body) | +| `/auction` body | `/auction` | Never | Never | That request only, from the body | +| Provider lookup | Navigation where source is due | That navigation's write | Same response (§10) | Next page load; same page if write done | +| After-response lookup | Navigation where source is due | Follow-up CAS update (§5.2.1) | Never on that response | Next page load | +| Batch sync | Batch endpoint | Its own write path | Never | After its write | + +### 5.6 Expiry and refresh + +- A UID with `exp <= now` is never served, mirrored or forwarded. +- An expired owned UID makes its source due; the owning provider is asked again + on the next navigation. No other provider and no browser input can refresh it. +- Expired UIDs stay in the row until the next write on any navigation, which + prunes them. Pruning alone never triggers a write. +- UIDs without `exp` and without refresh state are never refreshed; they stay until replaced or removed. +- A provider may keep private refresh state with a UID it writes (§6.3). Once + the state's `from` time passes, the source is due even though the UID is + still live, and the provider builds its refresh from the stored state + (`LookupInput::current`). For example, a token provider keeps the vendor's + refresh token and refreshes the token server side without any login input. + Vendor cookies are read first on every navigation; a usable cookie ID is + used instead of a refresh request. Refresh state that is empty, over 2048 + bytes or has no stated time is dropped and the UID is kept. +- A UID is refresh-due only while it is still live. A failing refresh is + therefore retried at most until the UID expires, and the provider's + `rate_limit_per_hour` also bounds it. +- A refresh reply may report an opt-out (`LookupOutcome::OptedOut`). It is + handled like a cookie opt-out: the provider's sources are cleared from the + row. +- A provider whose IDs have no vendor server may generate an ID instead of a + lookup (`IdentityProvider::generate`), for a due source when its cookies hold + none. The ID is validated and written like a lookup result; no request is + sent and no cookie is set. For example, a publisher common ID. Generation is + not rate-limited, and a source holding a live UID is not regenerated. +- A request may carry provider context (`ProviderRequest::context`) that is + never sent and is handed back with its response, for example a key that + decrypts the reply. + +### 5.7 Consent withdrawal + +- On any finalized request with the EC cookie: the tombstone is written outside + the page-load permit (§3.1); provider calls are not made; any lookup already + dispatched is discarded; no IDs are served and the mirror is expired. +- A concurrent navigation in another tab that tries to write collection results + after the tombstone fails its CAS and is not retried, so the tombstone wins. A + tombstone that loses its CAS to such a write retries, so the withdrawal still + wins. +- After fresh consent, the existing revive path creates a new row; previous IDs + are not restored. + +## 6. Type design + +### 6.1 Principles + +- **Every seam is a trait.** Five traits make the solution pluggable; each has + one production implementation and can be replaced in tests or future work + without touching shared policy. + + | Trait | Varies by | Production implementation | Other implementations | + | --------------------- | --------------------------- | ------------------------- | --------------------------------- | + | `IdentityProvider` | vendor | one per provider kind | test providers | + | `ProviderKind` | vendor (typed construction) | one per provider kind | — | + | `LookupTransport` | runtime / environment | `PlatformLookupTransport` | test transport; F4 mock transport | + | `IdentityCoordinator` | orchestration policy | `ProviderCoordinator

` | test fake for navigation tests | + | `EcKvStore` (exists) | KV backend | Fastly KV | existing in-memory test store | + +- **Static dispatch on the request path.** The coordinator, transport and + navigation code are generic. The new traits use native `async fn` (with + `#[allow(async_fn_in_trait)]`), not `async_trait`, so they add no boxed + futures; the existing `PlatformHttpClient` underneath keeps its own + `async_trait` boxing. The only trait objects are the configured providers: operators choose a + heterogeneous set, so each is boxed once at startup. `ProviderCoordinator

` + stays generic over `P`, so tests and single-provider builds use a concrete + type with no boxing. +- **Borrow per-request data.** Provider input borrows the request and the row + snapshot. Providers parse responses into types that borrow the response body + (`Cow<'body, str>`, `&'body RawValue`). Only entries that pass validation and + will be stored become owned. +- **Share config-derived identifiers.** `ProviderId` and `SourceDomain` wrap + `Arc`; clones made from config cost a reference count. `Serialize` and + `Deserialize` are implemented by hand: serializing writes the string + directly, and deserializing visits a `&str` and allocates the `Arc` + once. +- **Keep opaque JSON opaque in storage.** `ext` objects are stored as `RawJson`, + a `RawValue` newtype: never parsed into a map between provider response and + KV. They are parsed once, into `serde_json::Value`, only when an OpenRTB + request or header is built, because `crate::openrtb` already uses `Value` for + `ext`. This needs `serde_json`'s `raw_value` feature; no new crate is added. +- **Parse, don't validate.** Raw provider output converts into stored types + through `TryFrom`; past that boundary an invalid UID, agent type or match + method cannot be represented. +- **No micro-optimisation.** Small `Vec`s sized by provider or UID count are + fine; no arenas, no small-vector crates. + +### 6.2 Value types + +```rust +/// Unix time in whole seconds. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, derive_more::Display)] +#[serde(transparent)] +pub struct UnixSeconds(u64); + +/// Operator-chosen identifier of a configured provider instance. +/// +/// `Serialize` and `Deserialize` are implemented by hand (§6.1). +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, derive_more::Display)] +pub struct ProviderId(Arc); + +/// Normalized EID source domain. +/// +/// `Serialize` and `Deserialize` are implemented by hand (§6.1); +/// deserializing normalizes and rejects invalid domains. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, derive_more::Display)] +pub struct SourceDomain(Arc); + +/// A non-empty UID of at most [`MAX_STORED_UID_LENGTH`] (1024) bytes. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, derive_more::Display)] +#[serde(try_from = "String")] +pub struct UidValue(Box); + +impl TryFrom for UidValue { + type Error = EidValidationError; + // ... +} + +/// Non-negative OpenRTB agent type, including vendor-specific values. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(try_from = "i32")] +pub struct AgentType(i32); + +/// An opaque JSON object, compared by its bytes. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(transparent)] +pub struct RawJson(Box); + +impl PartialEq for RawJson { + fn eq(&self, other: &Self) -> bool { + self.0.get() == other.0.get() + } +} + +/// Why a provider value was rejected; counted per reason, never logged with +/// the value. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::Display)] +pub enum EidValidationError { + #[display("UID is empty")] + EmptyUid, + #[display("UID is too long")] + UidTooLong, + #[display("agent type is negative")] + NegativeAgentType, + #[display("match method is unknown")] + UnknownMatchMethod, + #[display("ext is not a JSON object within the size limit")] + InvalidExt, + #[display("source is not claimed by this provider")] + UnclaimedSource, + #[display("expiry is in the past")] + Expired, +} + +impl core::error::Error for EidValidationError {} + +/// OpenRTB 2.6 EID match method (`mm`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(into = "i32", try_from = "i32")] +pub enum MatchMethod { + Unknown, + NoMatching, + CookieSync, + Authenticated, + Observed, + Inference, +} + +/// When a stored UID stops being served. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Expiry { + /// Served until replaced or removed. + #[default] + Never, + /// Never served at or after this time. + At(UnixSeconds), +} + +impl Expiry { + /// Returns whether a UID with this expiry may be served at `now`. + #[must_use] + pub fn is_live(self, now: UnixSeconds) -> bool { + match self { + Self::Never => true, + Self::At(exp) => now < exp, + } + } + /// Returns whether this is [`Expiry::Never`]; used to omit `exp`. + #[must_use] + pub fn is_never(&self) -> bool { + matches!(self, Self::Never) + } +} +``` + +`Expiry` is stored as an optional `exp` field: absent for `Never`. + +```rust +/// Serde adapter storing [`Expiry`] as an optional `exp` number. +mod optional_exp { + pub fn serialize(expiry: &Expiry, serializer: S) -> Result { + match expiry { + Expiry::At(exp) => exp.serialize(serializer), + // Never reached: the field is skipped when `Expiry::is_never`. + Expiry::Never => serializer.serialize_none(), + } + } + + pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result { + Ok(Option::::deserialize(deserializer)?.map_or(Expiry::Never, Expiry::At)) + } +} +``` + +Sketches in this spec omit doc comments on self-describing fields and elide +bodies with `// ...`; implementations follow the documentation rules in +`AGENTS.md`. + +### 6.3 Stored row + +All additions are optional `serde` fields on schema `v: 1`. Rows written before +this change read unchanged. Rows written after it stay readable by older code +through the retained `uid` field, unless that `uid` (the first live provider +UID) is over 512 bytes: older code rejects such a row whole. + +The row also gains `opt_outs`, a map from provider ID to the Unix time until +which that provider's opt-out is remembered (30 days, §8.3). + +`uid` is rewritten on every write to the first live UID of `eids`, and new code +ignores it when `eids` is non-empty. A known limitation of staying on `v: 1` +(D12): during a rollout or rollback, older code reads only `uid`, so it can +serve that UID after its `exp`, and an older writer that re-saves the row drops +`eids` and `opt_outs`. A UID served past its `exp` and dropped `eids` end at the +next write by new code; a dropped opt-out is lost, so that provider is called +again. Stored values are built only by validation +(§8.2) or by deserializing rows TS wrote, so their fields are private to the +`ec` module and exposed through accessors. + +```rust +/// A partner ID record within a KV entry, keyed by source domain. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct KvPartnerId { + /// First live UID of `eids`, or the single UID for records written by + /// browser ingest, batch sync or older versions. + uid: String, + /// Full-fidelity entries written by a provider. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + eids: Vec, +} + +/// One stored EID entry with its EID-level fields. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct KvEid { + #[serde(default, skip_serializing_if = "Option::is_none")] + inserter: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + matcher: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + mm: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + ext: Option, + uids: Vec, +} + +/// One stored UID with its owner and expiry. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct KvUid { + id: UidValue, + /// Provider value, or the source's configured `openrtb_atype`. + atype: AgentType, + #[serde(default, skip_serializing_if = "Option::is_none")] + ext: Option, + /// Provider that wrote this UID; `None` for browser and batch UIDs. + #[serde(default, skip_serializing_if = "Option::is_none")] + owner: Option, + #[serde(default, rename = "exp", with = "optional_exp", skip_serializing_if = "Expiry::is_never")] + expiry: Expiry, + /// Owning provider's private refresh state (§5.6); never served. + #[serde(default, skip_serializing_if = "Option::is_none")] + refresh: Option, +} + +/// Refresh state a provider keeps with one of its UIDs. `Debug` hides +/// `state`, which may hold a credential. +#[derive(Clone, PartialEq, Serialize, Deserialize)] +pub struct KvRefresh { + /// Provider-defined state, opaque to the framework (at most 2048 bytes). + state: String, + /// When the owning provider should be asked to refresh the UID. + from: UnixSeconds, +} +``` + +Outbound EIDs (auction, mirror, `x-ts-eids`) are built from a UID's `id`, +`atype` and `ext` only, so `refresh` never leaves the row. Refresh state is +private: it is never served, mirrored, forwarded or shown. The authenticated +admin view (`/_ts/admin/ec/{id}`) shows each `refresh.state` as `[redacted]` +and keeps `from` visible. It is removed with the UID on opt-out, withdrawal or +expiry pruning. Rows without it read unchanged. + +A UID whose refresh state is empty or over 2048 bytes, or has no stated time, +keeps the UID and drops only the refresh state. + +The source domain stays the map key in `KvEntry::ids`, so one provider can own +several sources and each UID still records its owner. + +### 6.4 Reading the row + +Consumers read through borrowed views and build owned OpenRTB values once, at +the point they serialize a request or header: + +```rust +impl KvEntry { + /// Iterates entries for `bidstream_enabled` sources, yielding only live + /// UIDs and borrowing from the row. + pub fn live_eids<'row>( + &'row self, + registry: &'row PartnerRegistry, + now: UnixSeconds, + ) -> impl Iterator> + 'row; +} + +/// A borrowed EID entry: either a single legacy UID or a full-fidelity entry. +pub enum EidView<'row> { + Single { source: &'row str, uid: &'row str, atype: AgentType }, + Full { source: &'row str, eid: &'row KvEid, now: UnixSeconds }, +} + +impl From> for crate::openrtb::Eid { + // ... +} +``` + +The in-process `crate::openrtb::Eid` gains optional `inserter`, `matcher`, +`mm` and `ext` fields so full-fidelity entries reach bidders unchanged. + +## 7. Provider interface and crate boundary + +### 7.1 Where things live + +| Location | Contents | +| ------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `trusted-server-core` `ec::eid_types` | Value types and stored row types (§6). | +| `trusted-server-core` `ec::provider` | `IdentityProvider`, `ProviderKind`, the generic `build_kind` helper, input and output types, errors. No vendor code. | +| `trusted-server-core` `ec::coordinator` | `IdentityCoordinator`, `ProviderCoordinator

`, `LookupTransport`, `PlatformLookupTransport`, validation, merge, mirror encoding. | +| `crates/trusted-server-identity-providers` | One module per provider kind, each implementing `ProviderKind` and `IdentityProvider`, and one `build` function that matches `kind` to a `ProviderKind`. | +| `trusted-server-adapter-fastly` | Builds providers through the provider crate and a `ProviderCoordinator` at startup, and drives the navigation path with `PlatformLookupTransport`. | + +Dependencies point one way: the provider crate depends on core; core never +depends on the provider crate. Adding a provider means adding a module and one +`build` match arm in the provider crate. Shared consent, KV, cookie and auction +code never gains provider-specific branches. + +### 7.2 `IdentityProvider` + +Providers are pure: they build a request and parse a response. They never do +HTTP, KV, cookie or clock work. The trait is dyn-compatible so a configured set +can hold `Box`. + +```rust +/// Builds lookup requests and parses lookup responses for one identity provider. +/// +/// Implementations must not perform I/O or read the clock; the coordinator +/// executes requests and applies consent, transport, validation, expiry and +/// persistence policy. +pub trait IdentityProvider { + /// Describes which sources and EID shapes this provider can return. + fn capabilities(&self) -> &ProviderCapabilities; + + /// Builds the lookup request for the given due sources. + /// + /// # Errors + /// + /// Returns [`ProviderError::Request`] when the input cannot form a request. + fn build_request(&self, lookup: &LookupInput<'_>) -> Result>; + + /// Parses a provider response, borrowing from its body. + /// + /// # Errors + /// + /// Returns [`ProviderError::InvalidResponse`] when the body cannot be parsed + /// at all. Individually malformed entries are returned and dropped by the + /// coordinator's validation, not reported as an error. + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report>; + + /// Generates IDs for due sources instead of a lookup (§5.6). + /// + /// The default returns `None`: the provider makes a lookup. + fn generate(&self, lookup: &LookupInput<'_>) -> Option>> { + None + } +} + +impl IdentityProvider for Box

{ + // Delegates each method to `**self`. +} +``` + +Capabilities: + +```rust +/// What a provider can return, used by the coordinator to apply shared policy. +#[derive(Debug, Clone)] +pub struct ProviderCapabilities { + /// Sources this implementation can return and the input each needs. + pub sources: Box<[SourceCapability]>, + /// Limits and fields accepted from one response. + pub shape: EidShape, +} + +#[derive(Debug, Clone)] +pub struct SourceCapability { + pub source: SourceDomain, + pub input: SourceInput, +} + +/// Visitor input a source needs before it can be looked up. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceInput { + /// Works for any visitor. + Anonymous, + /// Needs a hashed email. Always skipped in V1. + HashedEmail, +} + +#[derive(Debug, Clone, Copy)] +pub struct EidShape { + pub max_eids: NonZeroUsize, + pub max_uids_per_eid: NonZeroUsize, + pub eid_level_fields: EidLevelFields, +} + +/// Whether `inserter`, `matcher`, `mm` and EID `ext` are kept. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EidLevelFields { + Preserved, + Ignored, +} +``` + +Input and request: + +```rust +/// Request-scoped data the coordinator lends to a provider. +#[derive(Debug, Clone, Copy)] +pub struct LookupInput<'a> { + pub ec_id: &'a str, + pub consent: &'a ConsentContext, + pub client: ClientSignals<'a>, + pub publisher_domain: &'a str, + /// Sources due for this provider, already filtered by consent and input. + pub due_sources: &'a [SourceDomain], + /// This provider's currently stored entries, borrowed from the row. + pub current: OwnedEntries<'a>, + /// Always `None` in V1. + pub hashed_email: Option<&'a HashedEmail>, +} + +#[derive(Debug, Clone, Copy)] +pub struct ClientSignals<'a> { + pub ip: Option, + pub user_agent: Option<&'a str>, +} + +/// An outbound request description. The coordinator, not the provider, sends it. +/// +/// `Debug` is implemented by hand and prints only the method and host, because +/// headers and query strings may carry secrets. Providers mark secret headers +/// with `HeaderValue::set_sensitive(true)`. +pub struct ProviderRequest { + pub method: http::Method, + pub url: url::Url, + pub headers: http::HeaderMap, + pub body: bytes::Bytes, + /// Never sent; handed back with the response (§5.6). + pub context: Option, +} +``` + +`OwnedEntries<'a>` is a borrowed view over the row that yields this provider's +`(&str, &KvEid)` pairs, keyed by the row's source strings. Provider secrets and settings are not in the +input: they are given to the provider once, when it is built (§7.3). + +PR3 ships `LookupInput` without the request's cookies. F1 (#1488) adds +`LookupInput::cookies` and `IdentityProvider::read_request_cookies`, which +returns a `CookieRead`; the cookie-first refresh (§5.6), cookie opt-outs (§8.3) +and scenarios 24 and 26 depend on it. + +Response and parsed output: + +```rust +#[derive(Debug, Clone, Copy)] +pub struct ProviderResponse<'body> { + pub status: http::StatusCode, + pub headers: &'body http::HeaderMap, + pub body: &'body [u8], + /// The request's `context`, handed back. + pub context: Option<&'body str>, +} + +#[derive(Debug)] +pub enum LookupOutcome<'body> { + Found(Vec>), + NoMatch, + /// The visitor opted out (§8.3). + OptedOut, +} + +/// One entry as returned by a provider, before validation. +#[derive(Debug)] +pub struct ProviderEid<'body> { + pub source: Cow<'body, str>, + pub inserter: Option>, + pub matcher: Option>, + pub mm: Option, + pub ext: Option<&'body RawValue>, + pub uids: Vec>, +} + +#[derive(Debug)] +pub struct ProviderUid<'body> { + pub id: Cow<'body, str>, + pub atype: Option, + pub ext: Option<&'body RawValue>, + pub expiry: ProviderExpiry, + /// Private refresh state to keep with the UID (§5.6). + pub refresh: Option>, +} + +#[derive(Debug)] +pub struct ProviderRefresh<'body> { + pub state: Cow<'body, str>, + /// When to refresh; resolved like `exp`. + pub from: ProviderExpiry, +} + +/// Expiry as a provider states it. The coordinator turns it into [`Expiry`] +/// using its own clock, so providers never read time. +#[derive(Debug, Clone, Copy)] +pub enum ProviderExpiry { + Unspecified, + At(UnixSeconds), + After(Duration), +} + +#[derive(Debug, derive_more::Display)] +pub enum ProviderError { + #[display("provider request could not be built")] + Request, + #[display("provider response could not be parsed")] + InvalidResponse, +} + +impl core::error::Error for ProviderError {} +``` + +Providers typically deserialize their wire format into private structs with +`#[serde(borrow)]` and map them into `ProviderEid<'body>`. + +### 7.3 `ProviderKind` and registration + +Construction is typed per provider; only the final value is erased. + +```rust +/// Typed constructor for one provider implementation. +pub trait ProviderKind { + /// Value of `kind` in `[[ec.identity_providers]]`. + const KIND: &'static str; + /// The provider-specific `settings` table. + type Settings: DeserializeOwned; + /// The constructed provider. + type Provider: IdentityProvider + 'static; + + /// Builds the provider from shared and provider-specific configuration. + /// + /// # Errors + /// + /// Returns [`ProviderConfigError::Invalid`] when the settings are unusable. + fn build( + common: &ProviderCommon<'_>, + settings: Self::Settings, + ) -> Result>; +} + +/// Shared configuration every provider receives at build time. +pub struct ProviderCommon<'a> { + pub endpoint: &'a url::Url, + pub sources: &'a [SourceDomain], + pub secrets: &'a ProviderSecrets, +} + +/// Parses `settings` for `K` and builds a boxed provider. Lives in core. +/// +/// `secrets` are resolved by the adapter at startup through the existing +/// secret path, from the key names in `config`. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError::Invalid`] when `settings` does not +/// deserialize into `K::Settings` or [`ProviderKind::build`] rejects it. +pub fn build_kind( + config: &IdentityProviderConfig, + secrets: &ProviderSecrets, +) -> Result, Report>; +``` + +The provider crate's single entry point, shown with the `pull-sync` arm that +PR5 adds (PR3 ships only the fallback arm): + +```rust +/// Builds the provider for a configured `kind`. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError::UnknownKind`] for an unregistered kind, or +/// any error from [`build_kind`]. +pub fn build( + config: &IdentityProviderConfig, + secrets: &ProviderSecrets, +) -> Result, Report> { + match config.kind() { + PullSync::KIND => build_kind::(config, secrets), + kind => Err(Report::new(ProviderConfigError::UnknownKind) + .attach(format!("kind `{kind}` is not registered"))), + } +} +``` + +`ProviderSecrets` holds `Redacted` values resolved through the existing +secret path and is passed only at build time. + +```rust +#[derive(Debug, derive_more::Display)] +pub enum ProviderConfigError { + #[display("provider kind is not registered")] + UnknownKind, + #[display("provider configuration is invalid")] + Invalid, +} + +impl core::error::Error for ProviderConfigError {} +``` + +`IdentityProviderConfig` is the deserialized `[[ec.identity_providers]]` entry +(§12.1). `HashedEmail` is a newtype over a validated lowercase hex SHA-256 +digest with no constructor reachable from request data in V1. + +## 8. Coordinator + +### 8.1 Responsibilities + +The coordinator owns, for every provider: + +1. **Eligibility.** Provider enabled; the EC's row exists and is not + tombstoned (an EC minted on this navigation, or a cookie EC with no row, is + not sent to vendors); at least one due source; sources needing `HashedEmail` removed. +2. **Consent.** The existing EID-persistence check passes, and the provider's + `tcf_vendor_id`, when configured, has vendor consent in the effective TCF + string. A failure skips that provider only, before any request is built. +3. **Destination.** The built URL must be `https` and its host must be in the + provider's configured `allowed_domains`. The URL comes from operator config; + browser data never selects a destination. A rejected destination skips the + provider and is counted. +4. **Transport.** All eligible requests are started through a + `LookupTransport` and gathered until all complete or the shared deadline + passes. Late responses are dropped (§8.1.1). +5. **Validation** (§8.2), **merge** (§8.3) and preparing the single write. +6. **Mirror** encoding (§10) from the row as it stands after the navigation. +7. **Logs and metrics** (§12). + +#### 8.1.1 `LookupTransport` + +The transport sends provider requests and gathers responses within a deadline. +It is the only coordinator dependency that touches the network, so swapping it +is how tests and the future debug mode (F4) run real providers and the real +coordinator against mock responses. + +```rust +/// Sends provider lookups and gathers their responses before a deadline. +#[allow(async_fn_in_trait)] +pub trait LookupTransport { + /// An in-flight request. + type Pending; + + /// Starts one request without waiting for its response. + /// + /// # Errors + /// + /// Returns [`TransportError::Send`] when the request cannot be started. + async fn start( + &self, + request: ProviderRequest, + deadline: Deadline, + ) -> Result>; + + /// Waits for in-flight requests until all finish or `deadline` passes. + /// + /// Requests still in flight at the deadline are abandoned and reported as + /// [`TransportError::Timeout`]. + async fn gather( + &self, + pending: Vec>, + deadline: Deadline, + ) -> Vec>>>; +} + +/// A value labelled with the provider it belongs to. +#[derive(Debug)] +pub struct Tagged { + pub provider: ProviderIndex, + pub value: T, +} + +/// An owned response; providers parse it by borrowing its body. +#[derive(Debug)] +pub struct TransportResponse { + pub status: http::StatusCode, + pub headers: http::HeaderMap, + pub body: bytes::Bytes, +} + +/// End of the shared lookup budget. +#[derive(Debug, Clone, Copy)] +pub struct Deadline(web_time::Instant); + +/// Position of a provider in the coordinator's configured list. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ProviderIndex(u16); + +#[derive(Debug, derive_more::Display)] +pub enum TransportError { + #[display("lookup request could not be sent")] + Send, + #[display("lookup did not complete before the deadline")] + Timeout, + #[display("lookup response exceeded the body limit")] + BodyTooLarge, +} + +impl core::error::Error for TransportError {} +``` + +`Deadline` wraps the `web_time::Instant` that `select_until` takes, because +core is built into the Cloudflare Worker (CI wasm clock lint). +`PlatformLookupTransport<'s>` borrows `RuntimeServices` and is a thin adapter +over the platform client: `start` calls `send_async`, and `gather` calls +`select_until` with the shared deadline while requests remain in flight, +stopping when none remain (it rejects an empty list) or it returns `Ok(None)`, +and tags each result with its provider. It adds no timing logic of its own. Response +bodies are capped at 64 KiB. + +#### 8.1.2 `IdentityCoordinator` + +The navigation path depends on the `IdentityCoordinator` trait, generic over +the coordinator and the transport. The trait covers the coordinator's whole +surface on a navigation: start lookups and collect them (PR B adds the mirror). +It does not plan the row change: the free function `write_plan::plan_write` +merges the results and plans it. Its production implementation is +`ProviderCoordinator

`. Tests of coordinator rules (validation, merge, +mirror) run `ProviderCoordinator` with test providers and a test transport; a +fake `IdentityCoordinator` is used only for navigation wiring tests, where +those rules are not under test. + +```rust +/// Coordinates server-side identity lookups for one navigation. +/// +/// Implementations own eligibility, consent, destination checks, the shared +/// time budget, validation, merging and mirror encoding. Providers never see +/// KV, cookies, HTTP or the clock. Implementations never write KV: the caller +/// plans the change with `plan_write` and spends the [`PageLoadWrite`] permit. +#[allow(async_fn_in_trait)] +pub trait IdentityCoordinator { + /// Starts lookups for every eligible provider without waiting. + /// + /// Returns empty pending lookups when nothing is due, consent does not + /// permit EID collection, or no provider is enabled. + async fn dispatch( + &self, + transport: &T, + lookup: &LookupContext<'_>, + ) -> PendingLookups; + + /// Waits for dispatched lookups until all finish or the deadline passes, + /// then parses and validates each response. + /// + /// Never fails: per-provider failures are recorded in [`LookupResults`]. + async fn collect( + &self, + transport: &T, + pending: PendingLookups, + ) -> LookupResults; + + /// Builds the mirror cookie from the row as it stands after this + /// navigation: the committed row, or the snapshot when no write landed. + fn mirror(&self, row: Option<&KvEntry>, consent: &ConsentContext, now: UnixSeconds) -> MirrorCookie; +} +``` + +Supporting types: + +```rust +/// Request-scoped inputs to [`IdentityCoordinator::dispatch`]. +pub struct LookupContext<'a> { + pub ec_id: &'a str, + pub snapshot: &'a EcKvSnapshot, + pub consent: &'a ConsentContext, + pub client: ClientSignals<'a>, + pub publisher_domain: &'a str, + /// Existing EC rate limiter, used only for `rate_limit_per_hour`. + pub rate_limiter: &'a dyn RateLimiter, + pub now: UnixSeconds, +} + +/// Lookups in flight for one navigation. Only the coordinator module can +/// create a non-empty value. `Default` (empty) and `Debug` (counts only) are +/// implemented by hand so they do not require bounds on `Pending`. +pub struct PendingLookups { + requests: Vec>, + deadline: Option, + /// Request time, for validating `exp` and resolving `ProviderExpiry::After`. + now: UnixSeconds, + skipped: Vec>, +} + +/// Validated results of one navigation's lookups. +#[derive(Debug, Default)] +pub struct LookupResults { + lookups: Vec, +} + +#[derive(Debug)] +pub struct ProviderLookup { + pub provider: ProviderId, + pub status: LookupStatus, + /// Validated entries per returned source; empty unless `status` is `Found`. + pub entries: Vec<(SourceDomain, Vec)>, +} + +/// One outcome per provider per navigation, used for logs and metrics. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LookupStatus { + Found { kept: u16, dropped: DropCounts }, + NoMatch, + Timeout, + HttpError(http::StatusCode), + TransportFailed, + InvalidResponse, + RequestFailed, + Skipped(SkipReason), +} + +/// Validation drops for one response, by reason. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct DropCounts { + pub invalid_uid: u16, + pub invalid_field: u16, + pub unclaimed_source: u16, + pub expired: u16, + pub over_limit: u16, + pub duplicate: u16, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SkipReason { + Consent, + NotDue, + Input, + Destination, + RateLimited, + OptedOut, +} + +/// Inputs to the free function `write_plan::plan_write`, the navigation's +/// planner (it is not a coordinator method). +/// +/// `PartnerIdUpdate` (today `pub(crate)` in `ec::kv`) becomes `pub` so this +/// public trait input does not leak a private type. +pub struct WritePlanInput<'a> { + pub ec_id: &'a str, + pub snapshot: &'a EcKvSnapshot, + pub new_entry: Option<&'a KvEntry>, + /// Updates from `ts-eids` and `sharedId`, already reduced by today's rules. + pub browser_ingest: &'a [PartnerIdUpdate], + pub lookups: &'a LookupResults, + /// For provider ownership. + pub registry: &'a PartnerRegistry, + pub now: UnixSeconds, +} + +/// What to do with the mirror cookie on this response. +#[derive(Debug)] +pub enum MirrorCookie { + Set(MirrorValue), + Expire, +} + +/// An encoded mirror value within the size limit (§10). +#[derive(Debug)] +pub struct MirrorValue(String); +``` + +The production coordinator: + +```rust +/// The production coordinator, built once at startup. +/// +/// Generic over the provider type: production uses the boxed default, tests +/// and single-provider builds use a concrete type. +pub struct ProviderCoordinator

> { + providers: Box<[Registered

]>, + /// Partner config: `openrtb_atype` for validation, `bidstream_enabled` + /// and config order for the mirror. + partners: Arc, + budget: Duration, +} + +/// A provider with the operator policy the coordinator applies to it. +struct Registered

{ + id: ProviderId, + policy: ProviderPolicy, + provider: P, +} + +/// Operator policy for one provider, validated at startup. +struct ProviderPolicy { + sources: Box<[SourceDomain]>, + allowed_domains: Box<[Box]>, + tcf_vendor_id: Option, + rate_limit_per_hour: Option, +} + +impl IdentityCoordinator for ProviderCoordinator

{ + // ... +} +``` + +`rate_limit_per_hour` is not a new traffic control: it carries over pull sync's +existing per-EC-hash, per-partner hourly limit (PR5), which needs the `RateLimiter` +state that pure providers cannot hold. Operators may also set it per provider +with the optional `rate_limit_per_hour` config field (§12.1), for example to +bound repeat lookups for sources a provider never resolves; it is unset by +default. Broader traffic controls remain future work (F2, F3). + +The navigation path, in order: + +```rust +async fn run_navigation(coordinator: &C, transport: &T, navigation: Navigation<'_>) -> Response +where + C: IdentityCoordinator, + T: LookupTransport, +{ + let Navigation { permit, request, graph, ec, .. } = navigation; + let pending = coordinator.dispatch(transport, &ec.lookup_context()).await; + let mut response = fetch_origin(request).await; + let lookups = coordinator.collect(transport, pending).await; + let mut row = ec.snapshot_entry(); + let change = permit + .confirm(response.status()) + .zip(write_plan::plan_write(&ec.write_plan(&lookups))); + if let Some((permit, change)) = change { + // One attempt, never retried (§9.2). + match graph.commit_page_load(permit, ec.id(), change) { + Ok(committed) => row = Some(committed), + Err(report) => ec.record_write_failure(&report), + } + } + ec.apply_cookies(&mut response, row.as_ref(), coordinator.mirror(row.as_ref(), ec.consent(), ec.now())); + response +} +``` + +### 8.2 Validation + +Validation keeps valid entries and drops only invalid ones. Each drop is +counted by reason. + +| Level | Rule | +| -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| EID | `source` normalizes to a source this provider claims in config; otherwise dropped. | +| EID | `inserter`/`matcher` at most 255 bytes; `mm` within OpenRTB values; `ext` a JSON object of at most 1 KiB. EID-level fields are dropped (not the entry) when `eid_level_fields` is `Ignored`. | +| UID | `id` non-empty after trimming and at most `MAX_STORED_UID_LENGTH` (1024) bytes; `ext` a JSON object of at most 1 KiB. | +| Both | An `ext` (EID or UID) nests at most 32 levels deep; otherwise it is invalid as above. | +| UID | Refresh `state` non-empty and at most 2048 bytes, with a stated `from`; otherwise the refresh state is dropped and the UID kept. `from` is resolved like `exp`. | +| UID | `atype` missing is filled from the source's `openrtb_atype`. | +| UID | `exp`, when present, must be greater than now; otherwise dropped. | +| Counts | Entries beyond `max_eids`, or UIDs beyond `max_uids_per_eid`, are dropped from the end. | +| Dedupe | UIDs with the same (`source`, `id`) keep the first occurrence, as in §11.1. | +| Row size | See §12.3 for rows over 16 KiB serialized. This navigation's provider UIDs are dropped, last source and last UID first, until the row without its browser UIDs fits. | + +An EID entry left with no UIDs is dropped. A response that cannot be parsed at +all is a failure for that provider; other providers and browser input are +unaffected. + +### 8.3 Merge into the row + +For each provider with a `Found` outcome and each source in its response: + +- The validated entries replace every UID of that source in the row, including + unowned ones (D8). Each written UID records `owner` and `exp`. +- Sources the provider claims but did not return stay unchanged. +- `NoMatch`, failures, timeouts and skipped providers leave the row unchanged. +- `OptedOut` (a cookie or a refresh reply) clears every source the provider + claims and records the opt-out in the row's `opt_outs` map for 30 days. Until + it lapses the provider is skipped as `skipped_opted_out` and not called, and + a result from a lookup dispatched before the opt-out was recorded is + discarded rather than written over it. Generated and cookie IDs merge like + `Found`. + +Browser ingest (§8.4) is applied in the same in-memory merge, before provider +results, so the write order never lets browser data overwrite provider data. + +### 8.4 Browser ingest (navigation only) + +Unchanged rules: registered sources only; one UID per source, the first valid +UID of the last matching entry, with `sharedId` applied last; no EID-level +fields; fill missing sources only; UIDs at most `MAX_UID_LENGTH` (512) bytes. +The planner prunes expired UIDs first, so a source holding only expired UIDs +can be filled by the same write. New rule: a provider-owned source is skipped +entirely. + +### 8.5 Batch sync + +Batch sync keeps its own write path. A record for a source claimed by an +enabled provider is rejected and counted as `provider_owned`; other records in +the same batch are processed as today. + +### 8.6 Disabled providers + +A disabled provider makes no outbound requests and is not considered for +eligibility, and its sources stop being provider-owned. UIDs it wrote earlier +keep serving until `exp`, or until replaced or removed; once pruned, browser +ingest may fill the source again. Its `tcf_vendor_id` still gates egress of +every ID for its sources, stored or sent by the browser, so disabling a +provider never releases its UIDs without that vendor's consent. Removing the +provider's entry does drop that gate (#1548). Other providers are unaffected. + +## 9. KV write counting, failure and retry + +### 9.1 Permit + +The write limit is enforced by types. A navigation receives a +`NavigationPermit`; it becomes a `PageLoadWrite` only when the final response +is not a redirect; the write consumes it. Neither type is `Clone` or +`Copy`, `classify` is called once, by the publisher-proxy route only, and +`commit_page_load` consumes the permit, so a navigation cannot write twice and +no other request can write at all. + +```rust +/// Issued to a request classified as a page-load navigation (§3.1). +#[derive(Debug)] +pub struct NavigationPermit(()); + +impl NavigationPermit { + /// Returns a permit when `request` is a top-level document navigation. + /// + /// Called once per request, by the router only. + #[must_use] + pub fn classify(request: &http::Request) -> Option; + + /// Confirms the permit once the final response status is known. + /// + /// Returns `None` for a redirect: the redirect target is the page load. + #[must_use] + pub fn confirm(self, status: http::StatusCode) -> Option; +} + +/// Proof that the current request may make the page load's single KV write. +#[derive(Debug)] +pub struct PageLoadWrite(()); + +// Also on the navigation permit: confirms it together with the permit for the +// after-response phase (§5.2.1). +// pub fn confirm_with_after_response(self, status: http::StatusCode) +// -> Option<(PageLoadWrite, AfterResponseWrite)>; + +/// Proof for the one follow-up update; consumed by `commit_after_response_lookups`. +#[derive(Debug)] +pub struct AfterResponseWrite(()); + +/// The single row change of a navigation, with its CAS precondition. +#[derive(Debug)] +pub enum RowChange { + /// Create a row that must not exist (new EC or orphan recovery). + Create(KvEntry), + /// Replace the row only if it is still at `expected`. + Update { entry: KvEntry, expected: Generation }, +} + +/// Why the single write did not land. +#[derive(Debug, derive_more::Display)] +pub enum KvWriteError { + #[display("row changed since it was read")] + Conflict, + #[display("row already exists")] + AlreadyExists, + #[display("KV store rejected the write")] + Store, +} + +impl core::error::Error for KvWriteError {} + +impl KvIdentityGraph { + /// Applies the navigation's single row change in one attempt. + /// + /// # Errors + /// + /// Returns [`KvWriteError`] on CAS conflict, an existing row on create, or + /// a store failure. Callers must not retry. + pub fn commit_page_load( + &self, + permit: PageLoadWrite, + ec_id: &str, + change: RowChange, + ) -> Result>; +} +``` + +`Generation` is a newtype over the KV generation captured in +`EcKvSnapshot::Present`. When the snapshot carries no generation, +`plan_write` returns `None` for updates and the navigation counts +`no_generation`: TS never overwrites a row blindly. + +After PR2 all browser-driven KV writes except withdrawal tombstones go through +`commit_page_load`: new EC creation, orphan recovery and browser ingest; PR3 +adds provider results. Withdrawal tombstones keep their existing path (§3.1). The post-send pull-sync write is the one exception until +PR5 removes it. + +### 9.2 Failure behaviour + +| Failure | Effect | +| --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | +| CAS conflict or store error | No retry. Row unchanged. Mirror built from the unchanged row. New EC cookie not set. Counted. | +| Tombstone write fails | Bounded CAS retry first. Mirror expired and no IDs served (consent gating). EC cookie kept so the next request retries it. | +| EC-ID collision on create | Same as a CAS conflict; the next navigation generates a new EC. | +| Provider timeout/outage/invalid | That provider's results are dropped; the write proceeds with everything else that is valid. | +| All providers fail | The write proceeds if browser ingest changed anything; otherwise no write. | +| Coordinator budget exceeded | Late responses dropped; the response is never held past the budget. | +| After-response row missing, withdrawn or EC changed | Nothing written (`no_row`); results dropped. | +| After-response write conflict | Results dropped; the source stays due and is retried on the next navigation. The sent response is unaffected. | + +Failed lookups are retried only by a later navigation where the source is +still due. + +## 10. Mirror cookie + +| Attribute | Value | +| --------- | -------------------------------------------------------------------------------------------- | +| Name | `__Host-ts-eids-mirror` | +| Flags | `Path=/; Secure; SameSite=Lax`; no `Domain`; not `HttpOnly` | +| Max-Age | Same as the EC cookie | +| Value | base64url of `{"v":1,"eids":[...]}`; each entry is an OpenRTB EID whose UIDs may carry `exp` | +| Size | Encoded value at most 3,584 bytes | + +- **Contents:** live, provider-owned UIDs of `bidstream_enabled` sources, with + their EID-level and UID-level fields. +- **Fill order:** sources in `[[ec.partners]]` config order; entries and UIDs in + stored order. Each UID is added if the encoded value still fits; otherwise it + is skipped and filling continues. KV keeps the complete set. +- **Set** on every consented navigation response from the row as it stands + after the navigation: the committed row, or the snapshot when no write + landed. Setting a cookie is not a KV write, and this restores a mirror the + browser lost. +- **Expired** (`Max-Age=0`) on any response where EID consent is absent or + withdrawn, whatever happened to the KV write, and on a navigation whose + mirror would be empty. +- **The server never reads it.** Readers on the page must drop UIDs whose `exp` + has passed and strip `exp` before use. + +## 11. Consumers + +All consumers read the row snapshot and gate by current consent and +`bidstream_enabled`, as today. None write KV. + +| Consumer | Behaviour | +| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `/auction` | Resolves the row to EIDs, full-fidelity for provider entries, dropping UIDs that are not live. Merges with the body EIDs (§11.1). Server IDs need no browser resend. | +| `/_ts/page-bids` | Same as `/auction`. | +| `x-ts-eids` header | Built from the merged set with the existing truncation flag. | +| `/_ts/api/v1/identify` | Includes provider UIDs that are live, under existing authentication and partner gating. | +| Integration proxies | Use the same resolver where they forward EIDs. | + +### 11.1 Merge for auctions + +1. KV entries first, then body EIDs. +2. EID entries are distinct when (`source`, `inserter`, `matcher`, `mm`) differ; + both are kept. +3. A UID whose `id` already appears under the same source is a duplicate; the + first occurrence (KV) keeps its fields. +4. Distinct UIDs are kept under their entry. + +## 12. Configuration, secrets and observability + +### 12.1 Configuration + +```toml +[ec] +# Shared time budget for all provider lookups on one navigation. +identity_budget_ms = 200 +# Budget for providers with collect = "after_response", run after the +# response is sent (§5.2.1). +identity_after_response_budget_ms = 2000 + +[[ec.identity_providers]] +id = "example-provider" +kind = "example-kind" +enabled = true +endpoint = "https://ids.example.com/v1/lookup" +allowed_domains = ["ids.example.com"] +# Each source needs an [[ec.partners]] entry and must be in the kind's capabilities. +sources = ["ids.example.com"] +# Optional: require TCF vendor consent for this provider. +tcf_vendor_id = 9999 +# Optional: "page" (default) or "after_response" (§5.2.1). +collect = "page" +# Optional: most lookups per EC hash per hour (§8.1); unset means unlimited. +rate_limit_per_hour = 4 + +# Secret Store key names, resolved like other secret references. +[ec.identity_providers.secrets] +token = "example_provider_token" + +[ec.identity_providers.settings] +# Provider-specific settings, parsed by the provider implementation. +``` + +Startup validation rejects: unknown `kind`; a source with no `[[ec.partners]]` +entry; a source outside the kind's capabilities; a source claimed by two +enabled providers; an `endpoint` host outside `allowed_domains`; placeholder +secrets in deploy validation. + +Validation also rejects an empty or duplicate provider `id`, an empty +`sources` list, a non-HTTPS `endpoint`, and identity budgets outside 1–1000 ms +(page) and 1–5000 ms (after-response). + +PR3 registers no provider kind, so deploy validation (`ts config validate`, +`push` and `diff`) rejects every enabled provider; a push therefore cannot +take a running service down. F1 replaces that rule: the CLI builds every +enabled provider of a registered kind, as the adapter does at startup, and +rejects enabled providers for a target adapter that does not run them, so +kind-specific setting errors fail before deploy. Core's own validation cannot +do this because provider kinds live outside core. A disabled provider's +`secrets.token` is not resolved. + +Secrets are Secret Store key references, resolved at startup into +`Redacted` values through the existing secret path. They are never written to cookies or logs, and source-controlled +examples use placeholders. + +### 12.2 Logs and metrics + +Per provider and navigation, one outcome: `found`, `no_match`, `timeout`, +`http_error`, `transport_error`, `request_error`, `invalid_response`, `skipped_no_row`, `skipped_consent`, +`skipped_not_due`, `skipped_input`, `skipped_rate_limited`, `skipped_opted_out`, +`destination_rejected`; plus the totals of UIDs kept and dropped. The page-load write emits no +outcome label in PR A: a skipped write is logged with its error (a conflict, +an existing row or a store error), and an update without a generation is +logged as `no_generation`. A navigation that runs the after-response phase +(§5.2.1) logs one write outcome: `written`, `unchanged`, `no_row`, `conflict` +or `failed`. The outcome log holds the provider ID, the outcome and the kept +and dropped totals; other logs hold the EC ID only in its existing truncated +form. Raw UIDs, EID values and encrypted envelopes are never logged. + +### 12.3 Compatibility and rollout + +PR A changes behaviour that operators and integrators can observe: + +- **EC minting.** A new EC is minted, and its row written, only on a + top-level publisher navigation: a `GET` with `Sec-Fetch-Dest: document`, + `Sec-Fetch-Mode: navigate`, no `prefetch` in `Sec-Purpose`, and a response + that is not a 301, 302, 303, 307 or 308 redirect. Clients without Fetch + Metadata get no EC, and integration proxies no longer mint one. +- **`user.id` withheld.** An EC minted on the current navigation is not sent + as `user.id`, and that navigation's auction carries no browser `ts-eids`, + until a later navigation finds its row. +- **Browser EIDs.** `/auction` no longer stores its body EIDs, and neither + `/auction` nor page-bids writes `ts-eids` or `sharedId`; the next navigation + ingests those cookies. +- **Withdrawal.** The EC cookie is expired only once every tombstone is + written or the row is proven absent; while a tombstone fails, the cookie is + kept so the next request retries. On main the cookie was expired first. +- **Vendor consent.** On Fastly, a provider's `tcf_vendor_id` gates every ID + of its sources at egress, browser-supplied ones included, and keeps doing so + while the provider is disabled (§8.6). +- **Row size cap.** Rows are capped at 16 KiB (`MAX_ENTRY_BYTES`). Batch sync + reports `row_full`. A row written before the cap is trimmed only when a + write also carries a real change: its largest stored records are dropped + first, so the change that triggered the write still lands. A provider + refresh that no longer fits keeps the record it would replace. +- **Fastly snapshot.** The final auction snapshot may wait up to 1 ms per + partially received body and reads on into a response whose headers arrived + by the cutoff. +- **Config blob rollback.** A blob that configures identity providers or sets + a non-default identity budget cannot be loaded by an older build; default + values are left out of the blob so an unchanged config stays compatible. +- **Row rollback.** Rows with a provider UID over 512 bytes fail to load on an + older build, and an older writer drops `eids` and `opt_outs` (§6.3). +- **Providers.** PR A registers no provider kind, so deploy validation and + every runtime refuse an enabled provider. + +## 13. PR breakdown + +### 13.1 Required + +The work below is delivered as two pull requests: + +- **PR A** — the work units PR1 to PR4, on top of the merged platform PR + #1433, which provides the deadline-bounded `select_until` that + `PlatformLookupTransport` uses. +- **PR B** — the work units PR5 and PR6, stacked on PR A. + +PR A alone keeps the old post-send pull-sync write as a separate channel +(§9), as it is on main; PR A registers no provider kind, so no provider write +adds to it. PR B removes it. Within each pull +request the units are separate, ordered commits so they stay reviewable. + +| PR | Title | Contents | Depends on | +| --- | ---------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------- | +| PR0 | Spec | This document. | — | +| PR1 | Full-fidelity EID storage and resolution | `ec::eid_types` value types (§6.2); `KvEid`/`KvUid` additive schema; EID-level fields on `crate::openrtb::Eid`; resolver drops UIDs that are not live; auction merge (§11.1); `x-ts-eids` and identify use it. No new writers. | PR0 | +| PR2 | Page-load write permit | Navigation classifier; `PageLoadWrite` and `commit_page_load`; single attempt; non-navigation requests read-only except withdrawal tombstones; browser ingest on navigation only; withdrawal handling (§3.1, §5.7); integration-proxy EC minting stops. | PR0 | +| PR3 | Provider trait and coordinator | `ec::provider` (`IdentityProvider`, `ProviderKind`), `ec::coordinator` (`IdentityCoordinator`, `ProviderCoordinator

`, `LookupTransport`, `PlatformLookupTransport`), provider crate with no kinds, config and validation, consent, allowlist, budgeted fan-out, validation, merge into the navigation write, provider-owned skip in ingest, metrics, example config. | PR1, PR2, #1433 | +| PR4 | Batch sync on provider-owned sources | Reject and count records for sources an enabled provider claims. | PR3 | +| PR5 | Pull sync as a provider | `pull-sync` kind built from existing `[[ec.partners]]` pull-sync settings with no config change; existing per-partner rate limit kept as provider-scoped eligibility; post-send dispatcher and completeness marker removed, leftover marker cookies expired. Behaviour change for existing pull-sync partners: their sources become provider-owned, so browser ingest skips them and, with PR4, batch sync rejects them; once PR6 lands their IDs are mirrored when `bidstream_enabled`. | PR3 | +| PR6 | Mirror cookie | Adds `IdentityCoordinator::mirror` (PR3 ships the trait without it); encoding, fill order, set and expiry rules (§10). | PR3 | + +### 13.2 Optional and future (Tech Lead decision) + +| ID | Task | Notes | Depends on | +| --- | --------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | +| O1 | tsjs reads the mirror (D6b) | Reads the mirror, drops expired UIDs, strips `exp`, passes entries to Prebid user IDs. Separate PR. | PR6 | +| F1 | Concrete provider implementations | R§5.1. One spec and one PR per vendor named in R§6, each a module and `build` arm in the provider crate. Also adds cookie reads to the core coordinator (`LookupInput::cookies`, `read_request_cookies`) and the kind- and adapter-aware deploy check (§12.1). | PR3 | +| F2 | Provider lookup traffic controls | R§5.2. Best-effort dedup per EC and provider, bounded no-match caching kept distinct from failures, refresh jitter. Strict cross-instance dedup is a separate decision. | PR3 | +| F3 | Per-provider cooldown | R§5.3. Pause after repeated failures, then probe for recovery. | PR3 | +| F4 | Provider debug mode | R§5.4. A mock `LookupTransport` runs real providers and the real coordinator against configurable responses; no real provider calls, no production KV or cookie mutation. | PR3 | + +## 14. Acceptance scenarios + +1. **First navigation, consent granted.** No EC. One provider is enabled and + due. → One KV create holding the browser-ingested UIDs; the EC cookie is + set; the provider is not called (§8.1). The next navigation finds the row, + calls the provider once and writes its UIDs (with owner and `exp`) in one + CAS update; the mirror is set on that response. +2. **One write per page load.** A navigation is followed by three `/auction` + calls, one `/_ts/page-bids` call and two integration-proxy calls, all + carrying new `ts-eids` content. → At most one KV write, on the navigation. +3. **Same-page and next-page availability.** Provider IDs written by the + navigation → the next page load's `/auction` contains them without the + browser resending them; a same-page `/auction` contains them only if it + read the row after the write. +4. **Slow provider.** Provider A answers in 50 ms, provider B in 500 ms, budget + 200 ms. → A's UIDs are written; B's response is dropped and counted as + `timeout`; the response is not held past the budget. +5. **Partial invalid response.** A provider returns three UIDs, one empty and + one over 1024 bytes. → One UID is stored; two drops are counted. +6. **Write failure.** The navigation's CAS fails. → No retry; mirror built from + the unchanged row; + on a new EC no EC cookie is set; the next navigation looks up again. +7. **Expiry.** A provider UID's `exp` passes. → It is excluded from `/auction`, + `x-ts-eids`, identify and the mirror; the next navigation asks only its + owning provider. +8. **No expiry.** A provider UID without `exp` → served until the provider + replaces it or consent is withdrawn; never refreshed unless it carries refresh state + (scenario 24). +9. **Withdrawal on a navigation.** → Tombstone written; no provider calls; EC + cookie and mirror expired; no IDs served. +10. **Withdrawal on `/auction`.** → Tombstone written (§3.1), EC cookie and + mirror expired, no IDs served. +11. **Tombstone beats a concurrent write.** Tab A writes a tombstone while tab + B's navigation is collecting provider results. → B's CAS fails and is not + retried; the provider IDs are never stored or served. +12. **Provider-owned source.** A browser `ts-eids` UID and a batch record arrive + for a source an enabled provider claims. → Browser UID ignored; batch + record rejected and counted; provider UIDs unchanged. +13. **Multi-source provider.** One provider returns entries for two claimed + sources with two UIDs each and EID-level fields. → Both sources stored with + all four UIDs and fields, each owned by that provider; `/auction` forwards + them at full fidelity. +14. **Disabled provider.** One of two providers is disabled. → No outbound + request to it; the other provider and browser ingest behave as before. +15. **Destination rule.** A provider builds a URL whose host is not in + `allowed_domains`. → No request is sent; counted as `destination_rejected`. +16. **Hashed-email source.** A source requires `HashedEmail`. → Skipped before + any call, counted as `skipped_input`. +17. **Mirror trimming.** Provider UIDs exceed 3,584 encoded bytes. → The mirror + holds the deterministic config-order fill of §10; KV holds all of them. +18. **No secrets or IDs in logs.** Every outcome above produces logs and metrics + without raw UIDs, envelopes or secrets. +19. **Nothing due.** A returning visitor with all owned UIDs live and no new + browser input. → No provider calls, no KV write. +20. **Redirect.** The navigation's origin response is a 302. → No KV write; the + redirect target navigation may write. +21. **Pull-sync partner after PR5.** A partner with pull sync enabled also + receives a browser `ts-eids` UID and a batch record. → Both are ignored or + rejected as for any provider-owned source; the pull-sync provider's UID is + stored and served. +22. **After-response provider.** Provider A is `collect = "page"` and answers + in 50 ms; provider B is `collect = "after_response"` and answers in + 900 ms; page budget 200 ms. → The response is sent without waiting for B; + the page write holds A's UIDs and browser ingest; after the response, one + CAS update adds B's UIDs with owner and `exp`; the next page load serves + them. +23. **After-response on a withdrawn or changed row.** The row is tombstoned, + or written by another request, between the page write and B's result. → + No row is created or revived; on a generation conflict B's results are + dropped and B is asked again on the next navigation. +24. **Provider refresh.** A provider-written UID is live until 5000 and holds + refresh state with `from` 900; the navigation is at 1000 and the request carries no usable vendor + cookie. → The source is + due; the provider's request is built from the stored state; the refreshed + UID and its new refresh state are written. With `from` 2000 the source is + not due and nothing is sent. +25. **Refresh reports an opt-out.** A refresh reply reports an opt-out. → The + provider's sources are cleared from the row; nothing is stored for them. +26. **Generated ID.** A provider that generates IDs has a due source and no + cookie ID. → No request is sent; the generated UID is validated and written + with its owner; on the next navigation the source holds a live UID and is + not regenerated. +27. **Refresh state is never served.** A row holds a UID with refresh state. + → `/auction`, the mirror cookie and `x-ts-eids` carry the UID's `id`, + `atype` and `ext` only. + +## Review log + +### Round 1 (2026-11-09, PR #1487) + +Reviewer findings and resolutions: + +- Withdrawal outside permitted navigations: tombstone moved outside the + permit with bounded CAS retry (§3.1). +- Stored provider UIDs served after vendor consent withdrawal: egress gated on + the owning provider's TCF vendor ID. +- Config blob and 1024-byte UIDs breaking rollback: identity defaults left + out of the blob; non-provider writers stay at 512 bytes. +- Unknown kinds skipped silently, duplicate provider IDs, deeply nested `ext`, + batch sync aborting on a full row, minted ECs sent to vendors: fixed. +- Negative caching (#1547), a maximum provider UID lifetime (#1548), + after-response retry (#1553), lenient readers (#1554) and client EIDs after + an opt-out (#1555): deferred to follow-up issues. + +### Round 2 (2026-11-10/11, PR #1487 at 889159cd6) + +- Opt-out race on after-response and confirmation-read adoption: results of + an opted-out provider are discarded (§8.3). +- Size cap: trimming alone plans no write; this navigation's browser UIDs are + dropped before any stored record. +- Lookups only for an EC with an existing consented row (§8.1). +- Deploy validation rejects enabled providers while no kind is registered; + disabled providers' secrets are not resolved (§12.1). +- Disabled providers keep their vendor-consent gate on stored UIDs; `ext` + nesting bounded to 32 levels; raw client IP passed to providers. +- Spec drift corrected (§5.3, §6.3, §7, §8.4, §9.2, §12, §13). + +### Round 3 (2026-11-12, PR #1487 at 4e7a6e02b) + +- Size cap: round 2's fix was incomplete. A row over the cap that gains an ID + now loses its largest stored records first, so the added ID lands; this + navigation's provider UIDs compete with the stored row only, and a refresh + that no longer fits keeps the record it replaces (§8.2, §12.3). +- A remembered opt-out clears only sources the row still stores, so the + after-response path skips its re-read once they are gone. +- Orphan lookup retargeting removed: dispatch never looks up an EC without a + row. Such lookups are counted as `skipped_no_row` (§12.2). +- Client-EID vendor filter tested at both auction call sites; main's direct + tombstone tests restored. +- Remaining drift corrected (§3.1, §5.3, §6.2, §8.1.2, §8.2, §8.6, §9.1, + §12.2, §12.3, §13.1, §14). Round 2's "Spec drift corrected" missed these. +- Deferred: deploy validation per adapter and kind (#1552), a full-row pull + sync loop (#1551). + +### Round 4 (2026-11-12, PR #1487 at 1ec3be72e, Claude Code (Opus 5.5) review agent) + +- No blocking findings. Re-checked the round-3 changes: the size cap's + over-cap trimming, provider UID dropping and stored-record restore; the + after-response re-read skip (a newly reported opt-out always clears at least + one source, since every provider claims one); `skipped_no_row` precedence + over consent; and that no caller still needed orphan retargeting. +- §5.2.1 now states that the after-response phase skips its read when no + result can change the row. + +### Round 5 (2026-11-12, PR #1487 at 7a291be23, Claude Code (Opus 5.5) review agent) + +Angle: data flow and edge cases end to end, from dispatch through +validation, the write planner, the page-load and after-response commits, to +egress. + +- Blocking, fixed: a disabled provider's `tcf_vendor_id` was dropped from the + egress gate when an enabled provider claimed the same source, so its stored + UIDs were served under the new owner's vendor consent only, or with none + when the new owner sets no vendor ID. This broke §8.6. Each partner now + keeps the vendor IDs of every configured provider claiming its source and + is withheld unless all have consent, in the bidstream, identify and the + client-EID filter. +- Non-blocking, no change: a live UID written by a provider that no longer + claims the source keeps the source not due for its new owner until the UID + expires (§5.6, §8.6 "keep serving until `exp`"). This is the intended + behaviour, already pinned by a test; the §4 "due source" entry reads + "owned UIDs" as UIDs on a provider-owned source. + +### Round 6 (2026-11-12, PR #1487 at 6719f19ea, Claude Code (Opus 5.5) review agent) + +Angle: adversarial. Malformed provider replies and browser input, config +combinations and adapter differences, on the tree after round 5's fix. + +- No blocking findings. Re-checked round 5's gate across enabled, disabled + and reassigned providers and all three egress paths; provider replies with + unclaimed, unnormalized or oversized sources, empty or oversized UIDs, + negative `atype`, deep or non-object `ext`, past expiry and unusable + refresh state; per-phase coordinators validating only their own providers + while deploy validation covers the whole set; zero or out-of-range budgets, + vendor IDs and rate limits rejected at parse. +- Non-blocking, no change: a client EID whose `source` fails normalization, + for example with surrounding whitespace or a port, matches no partner and + so passes the vendor filter unchanged. Such a source names no configured + partner; the page that sends it controls the request anyway. +- Non-blocking, no change: only the Fastly adapter builds the registry with + provider ownership and vendor gates (§12.3 "On Fastly"); per-adapter deploy + validation stays with #1552. diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 7b17751835..7bf4545a16 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -94,6 +94,12 @@ passphrase = "ec_passphrase" ec_store = "ec_identity_store" # Max concurrent partner pull-sync requests. pull_sync_concurrency = 3 +# Optional identity-provider budgets (see [[ec.identity_providers]] below). +# Every page-phase provider shares one budget (1-1000 ms): +# identity_budget_ms = 200 +# Budget for providers with collect = "after_response", which run after the +# response is sent and write their results in a follow-up write (1-5000 ms): +# identity_after_response_budget_ms = 2000 # Optional cluster-heuristic tuning (defaults shown): # cluster_trust_threshold = 10 # entries with cluster_size <= this are individual users # cluster_recheck_secs = 3600 # re-evaluate cluster_size after this many seconds @@ -115,6 +121,36 @@ pull_sync_concurrency = 3 # batch_rate_limit = 60 # max batch-sync requests/min (default 60) # pull_sync_enabled = false # default false +# Optional server-side identity providers (Fastly only), looked up on publisher +# navigations before the origin fetch. Their budgets are set in [ec] above. This +# release registers no provider kind, so `ts config push` rejects an enabled +# provider and every runtime refuses to start with one. +# +# [[ec.identity_providers]] +# id = "example-provider" +# kind = "example-kind" +# enabled = false +# endpoint = "https://ids.example.com/v1/lookup" +# The endpoint host must be listed here. +# allowed_domains = ["ids.example.com"] +# Each source needs an [[ec.partners]] entry with a matching source_domain and +# must be claimed by only one enabled provider. +# sources = ["ids.example.com"] +# Optional: require TCF vendor consent for lookups and for sending any ID of +# these sources to bidders, including browser IDs, even while disabled. +# tcf_vendor_id = 9999 +# "page" (default) collects before the response; "after_response" collects +# after it is sent, only for an EC issued on an earlier request. +# collect = "page" +# Optional: most lookups per EC hash (client IP) per hour. On Fastly this is +# enforced per minute with a minimum of one, so values below 60 behave like 60. +# rate_limit_per_hour = 60 +# Secret Store key name, resolved like other secret references. +# [ec.identity_providers.secrets] +# token = "example_provider_token" +# Provider-specific settings, parsed by the provider kind. +# [ec.identity_providers.settings] + # ============================================================================= # OPTIONAL — Core features (disabled/omitted by default)