From 25d96e9f7bc125e98562e98cfa211ec19e8e52b0 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 04:50:45 +0530 Subject: [PATCH 01/20] Run pull sync as an identity provider on navigations Pull-sync partners become provider-owned: browser ingest skips their sources and batch sync rejects them. The post-send dispatcher and the completeness marker are removed, and leftover marker cookies are expired. Signed-off-by: dhruv8sh --- .../trusted-server-adapter-fastly/src/app.rs | 6 +- .../trusted-server-adapter-fastly/src/main.rs | 75 +- crates/trusted-server-core/src/ec/cookies.rs | 13 + crates/trusted-server-core/src/ec/finalize.rs | 178 +-- crates/trusted-server-core/src/ec/kv.rs | 670 +-------- crates/trusted-server-core/src/ec/mod.rs | 91 +- .../trusted-server-core/src/ec/pull_sync.rs | 1247 ----------------- .../src/ec/pull_sync_marker.rs | 689 --------- crates/trusted-server-core/src/ec/registry.rs | 55 +- .../src/migration_guards.rs | 1 - crates/trusted-server-core/src/publisher.rs | 238 +--- crates/trusted-server-core/src/settings.rs | 35 +- .../src/lib.rs | 80 +- .../src/pull_sync.rs | 234 ++++ 14 files changed, 475 insertions(+), 3137 deletions(-) delete mode 100644 crates/trusted-server-core/src/ec/pull_sync.rs delete mode 100644 crates/trusted-server-core/src/ec/pull_sync_marker.rs create mode 100644 crates/trusted-server-identity-providers/src/pull_sync.rs diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index a37078e3c2..386f6e7466 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -373,6 +373,7 @@ pub(crate) struct EcFinalizeState { pub(crate) use_finalize_kv: bool, pub(crate) eids_cookie: Option, pub(crate) sharedid_cookie: Option, + #[cfg_attr(not(test), allow(dead_code))] pub(crate) is_real_browser: bool, /// Per-request services carried to the entry point so the pull-sync /// dispatcher can reuse the same platform HTTP client. @@ -461,8 +462,9 @@ fn joined_cookie_header(req: &Request) -> String { /// Captures the request signals for after-response identity lookups on a /// permitted publisher navigation. /// -/// Returns `None` when no after-response provider is enabled, the request has -/// no EC, or the identity graph is unavailable. +/// Returns `None` when no after-response provider (including a pull-sync +/// partner) is enabled, the request has no EC, or the identity graph is +/// unavailable. fn prepare_after_response_lookups( state: &AppState, req: &Request, diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index f2aff9de1a..acb0b411ea 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -26,9 +26,6 @@ use trusted_server_core::ec::finalize::ec_finalize_response; use trusted_server_core::ec::kv::KvIdentityGraph; use trusted_server_core::ec::page_load::NavigationPermit; use trusted_server_core::ec::provider::{ClientSignals, RequestCookies}; -use trusted_server_core::ec::pull_sync::{ - PullSyncContext, build_pull_sync_context, dispatch_pull_sync, -}; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::TrustedServerError; use trusted_server_core::integrations::RequestFilterEffects; @@ -570,13 +567,12 @@ fn edgezero_main( identity, &mut response, ) { - Ok(partner_registry) => { + Ok(_) => { send_edgezero_response( response, request_filter_effects.as_ref(), counters.as_ref(), ); - run_edgezero_pull_sync_after_send(settings, &partner_registry, &ec_state); if let Some(state) = app_state.as_deref() { run_after_response_lookups(state, &mut ec_state, after_response); } @@ -598,17 +594,12 @@ fn edgezero_main( identity, &mut response, ) { - Ok(partner_registry) => { + Ok(_) => { send_edgezero_response( response, request_filter_effects.as_ref(), counters.as_ref(), ); - run_edgezero_pull_sync_after_send( - &settings, - &partner_registry, - &ec_state, - ); return; } Err(e) => { @@ -720,33 +711,6 @@ fn apply_edgezero_ec_finalize( Ok(partner_registry) } -fn run_edgezero_pull_sync_after_send( - settings: &Settings, - partner_registry: &PartnerRegistry, - ec_state: &EcFinalizeState, -) { - if !ec_state.is_real_browser { - return; - } - - let prepared_context = build_pull_sync_context(&ec_state.ec_context, partner_registry); - let Some((context, kv)) = - prepare_pull_sync_after_send(prepared_context, || require_identity_graph(settings)) - else { - return; - }; - - let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); - dispatch_pull_sync( - settings, - &kv, - partner_registry, - &limiter, - &context, - &ec_state.services, - ); -} - /// Runs after-response identity lookups and applies their follow-up write. /// /// Called once the response has been sent, so provider latency never delays @@ -820,24 +784,6 @@ pub(crate) fn run_after_response_lookups( log::info!("After-response lookups: write outcome={outcome}"); } -fn prepare_pull_sync_after_send( - context: Option, - graph_factory: F, -) -> Option<(PullSyncContext, KvIdentityGraph)> -where - F: FnOnce() -> Result>, -{ - let context = context?; - let kv = match graph_factory() { - Ok(kv) => kv, - Err(err) => { - log::debug!("Pull sync: identity graph unavailable, skipping: {err:?}"); - return None; - } - }; - Some((context, kv)) -} - /// Sends a finalized `EdgeZero` response to the client. /// /// Streaming `EdgeZero` bodies commit headers first, then pipe chunks to Fastly's @@ -1280,23 +1226,6 @@ mod tests { } } - #[test] - fn pull_sync_noop_states_skip_post_send_graph_factory() { - let calls = std::cell::Cell::new(0); - let result = prepare_pull_sync_after_send(None, || { - calls.set(calls.get() + 1); - Err(Report::new(TrustedServerError::KvStore { - store_name: "unexpected".to_owned(), - message: "graph factory should not run".to_owned(), - })) - }); - assert!( - result.is_none(), - "a skipped pull-sync plan should return none" - ); - assert_eq!(calls.get(), 0, "should not invoke the graph factory"); - } - #[test] fn trace_path_classification_survives_core_request_conversion() { // `main` skips client-IP sanitization when the native path is a trace diff --git a/crates/trusted-server-core/src/ec/cookies.rs b/crates/trusted-server-core/src/ec/cookies.rs index ac0e0c05ba..71e5acb109 100644 --- a/crates/trusted-server-core/src/ec/cookies.rs +++ b/crates/trusted-server-core/src/ec/cookies.rs @@ -174,6 +174,19 @@ pub fn expire_ec_cookie(settings: &Settings, response: &mut Response) } } +/// Appends a raw `Set-Cookie` value, skipping values that are not valid +/// header values. +pub(crate) fn append_cookie(response: &mut Response, value: &str) { + match HeaderValue::from_str(value) { + Ok(value) => { + response.headers_mut().append(header::SET_COOKIE, value); + } + Err(err) => { + log::warn!("Skipping Set-Cookie: invalid header value: {err}"); + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 5c99b284d2..23043b3590 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -13,17 +13,17 @@ use http::Response; use super::consent::{ec_consent_granted, ec_consent_withdrawn}; use crate::consent::{ConsentContext, allows_eid_persistence}; +use crate::constants::COOKIE_TS_EC_PULL_COMPLETE; use crate::settings::Settings; use super::EcContext; -use super::cookies::{expire_ec_cookie, set_ec_cookie}; +use super::cookies::{append_cookie, expire_ec_cookie, set_ec_cookie}; use super::eid_types::UnixSeconds; use super::generation::{generate_ec_id, is_valid_ec_id}; use super::kv::{KvIdentityGraph, PartnerIdUpdate}; use super::kv_types::KvEntry; use super::page_load::{PageLoadWrite, RowChange}; use super::prebid_eids::collect_eid_updates; -use super::pull_sync_marker::{expire_marker, reconcile_marker}; use super::registry::PartnerRegistry; use super::write_plan::{WritePlanInput, plan_write}; use super::{EcKvSnapshot, current_timestamp, log_id}; @@ -62,7 +62,39 @@ pub fn ec_finalize_response( sharedid_cookie: Option<&str>, response: &mut Response, ) { - ec_context.validate_pull_sync_marker(settings, registry); + finalize_response_inner( + settings, + ec_context, + kv, + registry, + eids_cookie, + sharedid_cookie, + response, + ); + expire_retired_pull_marker(ec_context, response); +} + +/// Expires a leftover pull-sync completeness marker; the marker is retired. +fn expire_retired_pull_marker(ec_context: &EcContext, response: &mut Response) { + if ec_context.pull_marker_present() { + append_cookie( + response, + &format!( + "{COOKIE_TS_EC_PULL_COMPLETE}=; Path=/; Secure; SameSite=Lax; Max-Age=0; HttpOnly" + ), + ); + } +} + +fn finalize_response_inner( + settings: &Settings, + ec_context: &mut EcContext, + kv: Option<&KvIdentityGraph>, + registry: &PartnerRegistry, + eids_cookie: Option<&str>, + sharedid_cookie: Option<&str>, + response: &mut Response, +) { let (permit, after_response) = ec_context .take_navigation_permit() .and_then(|permit| permit.confirm_with_after_response(response.status())) @@ -71,12 +103,6 @@ pub fn ec_finalize_response( let consent_withdrawn = ec_consent_withdrawn(ec_context.consent()); if !consent_allows_ec { - // Expire the request-local marker independently of the EC cookie: a - // withdrawal must stop any pending pull-sync disclosure window. - if consent_withdrawn && ec_context.pull_sync_marker().was_present() { - expire_marker(ec_context.pull_sync_marker_mut(), response); - } - finalize_unusable_consent( settings, ec_context, @@ -126,8 +152,6 @@ pub fn ec_finalize_response( // mint a browser cookie with no backing identity-graph row. log::info!("Skipping generated EC response write because KV graph is unavailable"); } - - reconcile_pull_sync_marker(settings, registry, ec_context, response); } /// Shared inputs for the navigation's single write. @@ -281,24 +305,6 @@ fn update_returning( } } -fn reconcile_pull_sync_marker( - settings: &Settings, - registry: &PartnerRegistry, - ec_context: &mut EcContext, - response: &mut Response, -) { - let ec_id = ec_context.ec_value().map(str::to_owned); - let snapshot = ec_context.kv_snapshot().clone(); - reconcile_marker( - settings, - registry, - ec_id.as_deref(), - &snapshot, - ec_context.pull_sync_marker_mut(), - response, - ); -} - /// Withholds EID-derived KV updates when consent does not allow EID /// persistence, on top of the Purpose 1 (EC) gate `ec_finalize_response` /// already enforces before reaching this point. @@ -1121,59 +1127,6 @@ mod tests { ); } - #[test] - fn valid_marker_with_unread_snapshot_defers_orphan_recovery() { - let settings = create_test_settings(); - let orphaned_ec = sample_ec_id("orphn2"); - let consent = ConsentContext { - jurisdiction: Jurisdiction::NonRegulated, - source: ConsentSource::Cookie, - ..Default::default() - }; - let mut ec_context = EcContext::new_for_test_with_ip( - Some(orphaned_ec.clone()), - consent, - Some("192.0.2.10".to_owned()), - ); - ec_context.set_recovery_eligible(true); - ec_context.set_pull_sync_marker_for_test( - crate::ec::pull_sync_marker::PullSyncMarkerState::Valid { expires_at: 4_600 }, - ); - let mut partner = make_partner("pull.example.com"); - partner.pull_sync_enabled = true; - partner.pull_sync_url = Some("https://sync.example.com/pull".to_owned()); - partner.pull_sync_allowed_domains = vec!["sync.example.com".to_owned()]; - partner.ts_pull_token = Some(Redacted::new("pull-token".to_owned())); - let registry = PartnerRegistry::from_config(&[partner]).expect("should build registry"); - let graph = KvIdentityGraph::in_memory("test_store"); - ec_context.set_navigation_permit(NavigationPermit::for_test()); - let mut response = empty_response(); - - ec_finalize_response( - &settings, - &mut ec_context, - Some(&graph), - ®istry, - None, - None, - &mut response, - ); - - assert_eq!( - ec_context.ec_value(), - Some(orphaned_ec.as_str()), - "an unread snapshot should defer orphan rotation until marker expiry" - ); - assert!( - matches!(ec_context.kv_snapshot(), EcKvSnapshot::NotRead), - "a marker-skipped request should leave the snapshot unread" - ); - assert!( - response.headers().get(http::header::SET_COOKIE).is_none(), - "bounded orphan deferral should not rewrite browser identity state" - ); - } - #[test] fn finalize_returning_user_subresource_does_not_persist_eid_updates() { let settings = create_test_settings(); @@ -2239,9 +2192,7 @@ mod tests { }; let mut ec_context = make_context_with_consent(Some(&ec_id), Some(&ec_id), true, false, consent); - ec_context.set_pull_sync_marker_for_test( - crate::ec::pull_sync_marker::PullSyncMarkerState::Invalid, - ); + ec_context.set_pull_marker_present_for_test(true); ec_context.set_navigation_permit(NavigationPermit::for_test()); let graph = KvIdentityGraph::failing("unavailable-store"); let mut response = empty_response(); @@ -2279,59 +2230,6 @@ mod tests { ); } - #[test] - fn finalize_sets_marker_for_complete_pull_partner_snapshot() { - let settings = create_test_settings(); - let ec_id = sample_ec_id("compl1"); - let mut partner = make_partner("ssp.example.com"); - partner.pull_sync_enabled = true; - partner.pull_sync_url = Some("https://sync.example.com/pull".to_owned()); - partner.pull_sync_allowed_domains = vec!["sync.example.com".to_owned()]; - partner.ts_pull_token = Some(Redacted::new("pull-token".to_owned())); - let registry = PartnerRegistry::from_config(&[partner]).expect("should build registry"); - let mut ec_context = make_context( - Some(&ec_id), - Some(&ec_id), - true, - false, - Jurisdiction::NonRegulated, - ); - let mut entry = live_entry(); - entry.ids.insert( - "ssp.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId::single("partner-uid".to_owned()), - ); - ec_context.set_kv_snapshot(EcKvSnapshot::Present { - ec_id, - entry: Box::new(entry), - generation: Some(1), - }); - let mut response = empty_response(); - - ec_finalize_response( - &settings, - &mut ec_context, - None, - ®istry, - None, - None, - &mut response, - ); - - let cookies = response - .headers() - .get_all(http::header::SET_COOKIE) - .iter() - .filter_map(|value| value.to_str().ok()) - .collect::>(); - assert!( - cookies - .iter() - .any(|cookie| cookie.starts_with("ts-ec-pull-complete=v1.")), - "complete snapshot should issue the marker" - ); - } - #[test] fn explicit_withdrawal_without_marker_or_ec_cookie_does_not_set_cookie() { let settings = create_test_settings(); @@ -2370,9 +2268,7 @@ mod tests { ..Default::default() }; let mut ec_context = make_context_with_consent(None, None, false, false, consent); - ec_context.set_pull_sync_marker_for_test( - crate::ec::pull_sync_marker::PullSyncMarkerState::Invalid, - ); + ec_context.set_pull_marker_present_for_test(true); let mut response = empty_response(); ec_finalize_response( diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index 0a35fd8baa..fb78da178f 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -10,7 +10,6 @@ //! (organic request paths) or propagate them (sync endpoints). See the //! per-operation error handling policy in the spec §7.5. -use std::collections::BTreeMap; use std::fmt; use std::sync::Arc; use std::time::Duration; @@ -92,32 +91,6 @@ impl PartnerIdUpdate { } } -pub(crate) fn apply_partner_id_updates(entry: &mut KvEntry, updates: &[PartnerIdUpdate]) -> bool { - let mut latest_updates = BTreeMap::new(); - for update in updates { - latest_updates.insert(update.partner_id.as_str(), update.uid.as_str()); - } - - let mut changed = false; - for (partner_id, uid) in latest_updates { - if entry - .ids - .get(partner_id) - .is_some_and(|existing| existing.uid == uid) - { - continue; - } - - entry.ids.insert( - partner_id.to_owned(), - super::kv_types::KvPartnerId::single(uid.to_owned()), - ); - changed = true; - } - - changed -} - /// EC identity graph on top of the platform KV store primitives. /// /// Each EC ID (`{64hex}.{6alnum}`) maps to a JSON-encoded [`KvEntry`] @@ -555,183 +528,6 @@ impl KvIdentityGraph { ))) } - /// Merges partner IDs using request-scoped persisted state as the first CAS input. - /// - /// A caller-supplied `Present` snapshot is *proof* that the row exists — - /// an earlier authoritative read or a confirmed write in the same - /// request. Partner-ID enrichment is - /// best effort, so a refresh that reads absent or unreadable never - /// downgrades that proof: the update is skipped and logged, and the proven - /// snapshot is returned so cookie issuance continues from the confirmed - /// write. A failed *write* still reports [`EcKvSnapshot::Failed`] — that is - /// an operation failure, not an ambiguous read. - /// - /// A caller-supplied `Missing` snapshot is revalidated once before the - /// updates are dropped, because a point read on an eventually-consistent - /// store cannot prove absence and routes without orphan recovery have no - /// later chance to retry. - pub(crate) fn upsert_partner_ids_from_snapshot( - &self, - ec_id: &str, - updates: &[PartnerIdUpdate], - snapshot: EcKvSnapshot, - ) -> EcKvSnapshot { - if updates.is_empty() { - return snapshot; - } - - // Existence proof carried by the incoming snapshot. Retained across - // every refresh so a best-effort enrichment read can never retract it. - let proven = match snapshot { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - .. - } if snapshot_id == ec_id => Some(snapshot.clone()), - _ => None, - }; - - // Resolve the initial usable snapshot without spending a CAS attempt. - // Every snapshot except a usable `Present` for this EC ID and a read - // that already failed is refreshed once. A `Missing` recorded earlier - // in the request is not proof of absence: edge KV point reads are - // eventually consistent, and named routes such as `/auction` and - // `/_ts/page-bids` never run orphan recovery, so short-circuiting on a - // stale miss there would drop the request's collected partner IDs - // outright. A refresh that still misses keeps the no-create behavior. - // A `Failed` read is returned as-is — the hot path never retries a - // lookup that already errored. Resolving here keeps all - // `MAX_CAS_RETRIES` iterations available for actual writes. - let mut current = match snapshot { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - generation: Some(_), - .. - } if snapshot_id == ec_id => snapshot, - EcKvSnapshot::Failed { - ec_id: ref snapshot_id, - } if snapshot_id == ec_id => return snapshot, - _ => self.load_snapshot(ec_id), - }; - - for _attempt in 0..MAX_CAS_RETRIES { - let (mut entry, generation) = match current { - EcKvSnapshot::Present { - ec_id: ref snapshot_id, - ref entry, - generation: Some(generation), - } if snapshot_id == ec_id => (entry.as_ref().clone(), generation), - // A refreshed read that is absent or unreadable is authoritative - // for this write: never create or overwrite a missing root. It - // is not authoritative for *existence* though, so a snapshot - // that already proved the row exists survives the refresh. - EcKvSnapshot::Missing { .. } | EcKvSnapshot::Failed { .. } => { - return Self::keep_proven(ec_id, current, proven.as_ref()); - } - // `load_snapshot` never yields `NotRead` or a generation-less - // `Present`; fail closed if that invariant is ever violated. - EcKvSnapshot::Present { .. } | EcKvSnapshot::NotRead => { - return EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }; - } - }; - - if !entry.consent.ok { - return current; - } - if !apply_partner_id_updates(&mut entry, updates) { - return current; - } - let (body, meta_str) = - match Self::serialize_entry_unless_full(&entry, self.store_name()) { - Ok(SerializedEntry::Fits { body, meta }) => (body, meta), - Ok(SerializedEntry::TooLarge(bytes)) => { - log::warn!( - "snapshot partner upsert for '{}' skipped: row would be {bytes} bytes \ - (limit {MAX_ENTRY_BYTES})", - log_id(ec_id) - ); - return EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }; - } - Err(err) => { - log::warn!( - "snapshot partner upsert for '{}' could not serialize the row: {err:?}", - log_id(ec_id) - ); - return EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }; - } - }; - match self.write_entry( - ec_id, - &body, - &meta_str, - ENTRY_TTL, - EcKvWriteMode::IfGenerationMatch(generation), - ) { - Ok(EcKvWriteOutcome::Written) => { - return EcKvSnapshot::Present { - ec_id: ec_id.to_owned(), - entry: Box::new(entry), - generation: None, - }; - } - Ok(EcKvWriteOutcome::PreconditionFailed) => { - current = self.load_snapshot(ec_id); - } - Err(err) => { - log::warn!( - "snapshot partner upsert failed for '{}': {err:?}", - log_id(ec_id) - ); - return EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - }; - } - } - } - - log::warn!( - "snapshot partner upsert for '{}': CAS conflict after {MAX_CAS_RETRIES} retries; \ - {} partner updates were not persisted", - log_id(ec_id), - updates.len(), - ); - EcKvSnapshot::Failed { - ec_id: ec_id.to_owned(), - } - } - - /// Returns `proven` instead of a read outcome that cannot disprove it. - /// - /// Partner-ID enrichment is best effort. When the caller already held proof - /// that the row exists — the page-load write's `Add`-confirmed create or an - /// earlier authoritative read in the same request — a refresh that misses or fails - /// says nothing about existence, so the proof is kept and the skipped - /// update is logged. Write failures are *not* routed here: they are real - /// operation failures and stay [`EcKvSnapshot::Failed`] so callers can - /// report them. - fn keep_proven( - ec_id: &str, - downgraded: EcKvSnapshot, - proven: Option<&EcKvSnapshot>, - ) -> EcKvSnapshot { - match proven { - Some(proven) => { - log::warn!( - "snapshot partner upsert skipped for '{}': refresh was not authoritative; \ - keeping the confirmed row", - log_id(ec_id) - ); - proven.clone() - } - None => downgraded, - } - } - /// Atomically merges a partner ID into the existing entry. /// /// Uses CAS (generation markers) to avoid clobbering concurrent writes @@ -1523,7 +1319,7 @@ mod tests { use super::*; use crate::ec::checked_current_timestamp; use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, UidValue, UnixSeconds}; - use crate::ec::kv_backend::test_support::{InMemoryEcKv, WriteFailingEcKv}; + use crate::ec::kv_backend::test_support::InMemoryEcKv; use crate::ec::kv_types::{KvEid, KvUid}; fn snapshot_ec_id() -> String { @@ -1854,93 +1650,6 @@ mod tests { ); } - #[test] - fn apply_partner_id_updates_returns_unchanged_for_empty_updates() { - let mut entry = live_entry(); - - let changed = apply_partner_id_updates(&mut entry, &[]); - - assert!(!changed, "should not change entry for empty updates"); - assert!(entry.ids.is_empty(), "should not add partner IDs"); - } - - #[test] - fn apply_partner_id_updates_skips_matching_existing_uid() { - let mut entry = live_entry(); - entry.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId::single("uid-1".to_owned()), - ); - let updates = vec![PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let changed = apply_partner_id_updates(&mut entry, &updates); - - assert!(!changed, "should not change when UID already matches"); - assert_eq!(entry.ids["ssp_x"].uid, "uid-1"); - } - - #[test] - fn apply_partner_id_updates_inserts_new_partner_uid() { - let mut entry = live_entry(); - let updates = vec![PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let changed = apply_partner_id_updates(&mut entry, &updates); - - assert!(changed, "should report changed entry"); - assert_eq!(entry.ids["ssp_x"].uid, "uid-1"); - } - - #[test] - fn apply_partner_id_updates_overwrites_different_uid() { - let mut entry = live_entry(); - entry.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId::single("old-uid".to_owned()), - ); - let updates = vec![PartnerIdUpdate::new("ssp_x", "new-uid")]; - - let changed = apply_partner_id_updates(&mut entry, &updates); - - assert!(changed, "should report changed entry"); - assert_eq!(entry.ids["ssp_x"].uid, "new-uid"); - } - - #[test] - fn apply_partner_id_updates_applies_multiple_updates() { - let mut entry = live_entry(); - let updates = vec![ - PartnerIdUpdate::new("ssp_x", "uid-x"), - PartnerIdUpdate::new("ssp_y", "uid-y"), - ]; - - let changed = apply_partner_id_updates(&mut entry, &updates); - - assert!(changed, "should report changed entry"); - assert_eq!(entry.ids["ssp_x"].uid, "uid-x"); - assert_eq!(entry.ids["ssp_y"].uid, "uid-y"); - } - - #[test] - fn apply_partner_id_updates_uses_last_duplicate_value() { - let mut entry = live_entry(); - entry.ids.insert( - "ssp_x".to_owned(), - crate::ec::kv_types::KvPartnerId::single("original".to_owned()), - ); - let updates = vec![ - PartnerIdUpdate::new("ssp_x", "intermediate"), - PartnerIdUpdate::new("ssp_x", "original"), - ]; - - let changed = apply_partner_id_updates(&mut entry, &updates); - - assert!( - !changed, - "should not write when the final duplicate value matches existing state" - ); - assert_eq!(entry.ids["ssp_x"].uid, "original"); - } - #[test] fn evaluate_cluster_returns_stored_value_without_store_io() { let kv = KvIdentityGraph::failing("nonexistent_store_for_cluster_cache_test"); @@ -2353,51 +2062,6 @@ mod tests { ); } - #[test] - fn snapshot_bulk_upsert_returns_persisted_entry_without_stale_generation() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - kv.create(&ec_id, &live_entry()).expect("should create"); - let snapshot = kv.load_snapshot(&ec_id); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = kv.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - - let entry = outcome - .entry_for(&ec_id) - .expect("should retain persisted entry"); - assert_eq!( - entry.ids.get("ssp_x").map(|id| id.uid.as_str()), - Some("uid-1") - ); - assert_eq!( - outcome.generation_for(&ec_id), - None, - "backend does not return the post-write generation" - ); - } - - #[test] - fn snapshot_bulk_upsert_does_not_create_missing_root() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = kv.upsert_partner_ids_from_snapshot( - &ec_id, - &updates, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); - - assert!( - matches!(outcome, EcKvSnapshot::Missing { .. }), - "missing snapshot should remain missing" - ); - assert!(kv.get(&ec_id).expect("should read store").is_none()); - } - #[test] fn write_withdrawal_tombstone_overwrites_live_entry() { let kv = KvIdentityGraph::in_memory("test_store"); @@ -2685,274 +2349,6 @@ mod tests { } } - #[test] - fn snapshot_upsert_with_generation_writes_without_reading() { - let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let graph = KvIdentityGraph::counting("counting-store", lookups.clone()); - let ec_id = snapshot_ec_id(); - graph.create(&ec_id, &live_entry()).expect("should seed"); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: Some(1), - }; - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - - assert_eq!( - lookups.load(std::sync::atomic::Ordering::Relaxed), - 0, - "a usable generation must avoid the initial read" - ); - assert_eq!( - outcome - .entry_for(&ec_id) - .and_then(|entry| entry.ids.get("ssp_x")) - .map(|id| id.uid.as_str()), - Some("uid-1") - ); - assert_eq!( - outcome.generation_for(&ec_id), - None, - "a successful write should clear the stale generation" - ); - } - - #[test] - fn snapshot_upsert_unchanged_updates_preserve_generation() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - let mut seeded = live_entry(); - apply_partner_id_updates(&mut seeded, &[PartnerIdUpdate::new("ssp_x", "uid-1")]); - kv.create(&ec_id, &seeded).expect("should seed"); - let snapshot = kv.load_snapshot(&ec_id); - assert_eq!( - snapshot.generation_for(&ec_id), - Some(1), - "seeded snapshot should carry its stored generation" - ); - - let outcome = kv.upsert_partner_ids_from_snapshot( - &ec_id, - &[PartnerIdUpdate::new("ssp_x", "uid-1")], - snapshot, - ); - - assert_eq!( - outcome.generation_for(&ec_id), - Some(1), - "an unchanged merge preserves the usable generation and performs no write" - ); - } - - #[test] - fn snapshot_upsert_refreshes_unavailable_generation_exactly_once() { - let lookups = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let graph = KvIdentityGraph::counting("counting-store", lookups.clone()); - let ec_id = snapshot_ec_id(); - graph.create(&ec_id, &live_entry()).expect("should seed"); - // Finalize-written style snapshot: entry known, generation unavailable. - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: None, - }; - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - - assert_eq!( - lookups.load(std::sync::atomic::Ordering::Relaxed), - 1, - "an unavailable generation refreshes exactly once before CAS" - ); - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|e| e.ids.contains_key("ssp_x")), - "refreshing an unavailable generation should persist the partner update" - ); - } - - #[test] - fn snapshot_upsert_gen_unavailable_survives_four_conflicts_then_writes() { - // A generation-unavailable snapshot (finalize-written style) refreshes - // once to obtain a usable generation. That refresh must not consume a - // CAS attempt, so all five write attempts remain: four conflicts - // followed by a successful fifth write still persist the update. - let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(4, false)); - let ec_id = snapshot_ec_id(); - graph.create(&ec_id, &live_entry()).expect("should seed"); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: None, - }; - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - - assert_eq!( - outcome - .entry_for(&ec_id) - .and_then(|entry| entry.ids.get("ssp_x")) - .map(|id| id.uid.as_str()), - Some("uid-1"), - "the fifth CAS attempt must still succeed after a refresh and four conflicts" - ); - } - - #[test] - fn snapshot_upsert_cas_conflict_remerges_concurrent_data() { - let graph = KvIdentityGraph::new(ConflictInjectingEcKv::new(1, true)); - let ec_id = snapshot_ec_id(); - graph.create(&ec_id, &live_entry()).expect("should seed"); - let snapshot = graph.load_snapshot(&ec_id); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - - let entry = outcome - .entry_for(&ec_id) - .expect("should persist re-merged entry"); - assert_eq!( - entry.ids.get("ssp_x").map(|id| id.uid.as_str()), - Some("uid-1"), - "conflict must re-merge our update onto the concurrently revived row" - ); - assert!(entry.consent.ok, "concurrent revive keeps the row live"); - } - - #[test] - fn snapshot_upsert_revalidates_transient_missing_and_persists() { - // An eventually-consistent point read earlier in the request missed a - // row that exists. Named routes such as `/auction` never run orphan - // recovery, so this refresh is the request's only chance to persist the - // collected partner IDs. - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - kv.create(&ec_id, &live_entry()).expect("should seed live"); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = kv.upsert_partner_ids_from_snapshot( - &ec_id, - &updates, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); - - assert_eq!( - outcome - .entry_for(&ec_id) - .and_then(|entry| entry.ids.get("ssp_x").map(|id| id.uid.clone())), - Some("uid-1".to_owned()), - "a stale miss must be revalidated before the updates are dropped" - ); - let (stored, _) = kv - .get(&ec_id) - .expect("should read store") - .expect("should keep the row"); - assert_eq!( - stored.ids.get("ssp_x").map(|id| id.uid.as_str()), - Some("uid-1"), - "the revalidated update must reach the store" - ); - } - - #[test] - fn snapshot_upsert_confirmed_missing_still_never_creates() { - // Revalidation only changes what a *stale* miss does. A row that is - // genuinely absent on the refresh must stay absent. - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = kv.upsert_partner_ids_from_snapshot( - &ec_id, - &updates, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - ); - - assert!( - matches!(outcome, EcKvSnapshot::Missing { .. }), - "a confirmed miss must stay missing" - ); - assert!( - kv.get(&ec_id).expect("should read store").is_none(), - "must not create a root entry for a missing key" - ); - } - - #[test] - fn snapshot_upsert_failed_snapshot_is_not_revalidated() { - // A lookup that already errored is not retried on the hot path, even - // though the row exists. - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - kv.create(&ec_id, &live_entry()).expect("should seed live"); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = kv.upsert_partner_ids_from_snapshot( - &ec_id, - &updates, - EcKvSnapshot::Failed { - ec_id: ec_id.clone(), - }, - ); - - assert!( - matches!(outcome, EcKvSnapshot::Failed { .. }), - "a failed lookup must not be retried by partner enrichment" - ); - } - - #[test] - fn snapshot_upsert_rejects_tombstone() { - let kv = KvIdentityGraph::in_memory("test_store"); - let ec_id = snapshot_ec_id(); - kv.create(&ec_id, &KvEntry::tombstone(1000)) - .expect("should seed tombstone"); - let snapshot = kv.load_snapshot(&ec_id); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = kv.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - - assert!( - outcome - .entry_for(&ec_id) - .is_some_and(|entry| entry.ids.is_empty()), - "a tombstone must reject partner enrichment" - ); - let (stored, _) = kv - .get(&ec_id) - .expect("should read store") - .expect("should keep the tombstone"); - assert!(stored.ids.is_empty(), "no update should reach the store"); - } - - #[test] - fn snapshot_upsert_store_failure_returns_failed_not_request_local() { - let graph = KvIdentityGraph::new(WriteFailingEcKv::new()); - let ec_id = snapshot_ec_id(); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: Some(1), - }; - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, snapshot); - - assert!( - matches!(outcome, EcKvSnapshot::Failed { .. }), - "a store write failure must not claim request-local IDs were persisted" - ); - } - #[test] fn upsert_partner_id_rejects_tombstone() { let graph = KvIdentityGraph::in_memory("test-store"); @@ -3003,70 +2399,6 @@ mod tests { ); } - #[test] - fn snapshot_upsert_keeps_add_confirmed_present_when_refresh_misses() { - // The page-load write records its successful `Add` create as `Present` - // without a generation. EID ingestion refreshes that snapshot to obtain one; on an - // eventually-consistent store the refresh can miss. Enrichment is best - // effort, so the miss must not retract the confirmed create — otherwise - // finalization suppresses the `ts-ec` cookie for a root that was written. - let graph = KvIdentityGraph::in_memory("empty-store"); - let ec_id = snapshot_ec_id(); - let add_confirmed = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: None, - }; - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, add_confirmed); - - assert!( - outcome.entry_for(&ec_id).is_some(), - "an Add-confirmed row must survive a non-authoritative refresh miss" - ); - assert!( - graph.get(&ec_id).expect("should read store").is_none(), - "a missed refresh must not create or overwrite a root" - ); - } - - #[test] - fn snapshot_upsert_keeps_add_confirmed_present_when_refresh_fails() { - let graph = KvIdentityGraph::failing("failing-store"); - let ec_id = snapshot_ec_id(); - let add_confirmed = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(live_entry()), - generation: None, - }; - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, add_confirmed); - - assert!( - outcome.entry_for(&ec_id).is_some(), - "a read failure is not evidence of absence and must not retract the create" - ); - } - - #[test] - fn snapshot_upsert_without_proof_still_reports_a_refresh_miss() { - // No prior proof of existence: a `NotRead` snapshot that refreshes into - // a miss must stay `Missing` so finalization can run orphan recovery. - let graph = KvIdentityGraph::in_memory("empty-store"); - let ec_id = snapshot_ec_id(); - let updates = [PartnerIdUpdate::new("ssp_x", "uid-1")]; - - let outcome = - graph.upsert_partner_ids_from_snapshot(&ec_id, &updates, EcKvSnapshot::NotRead); - - assert!( - matches!(outcome, EcKvSnapshot::Missing { .. }), - "an unproven refresh miss must remain a miss" - ); - } - #[test] fn a_store_error_never_carries_the_whole_identifier() { // Every message in this module goes through `log_id`, so a report that diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index b227f1dbc8..e1f750f6d1 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -29,7 +29,6 @@ //! - [`eids`] — Shared EID resolution and formatting helpers //! - [`eid_types`] — Value types for full-fidelity provider EIDs //! - [`batch_sync`] — S2S batch sync endpoint (`POST /_ts/api/v1/batch-sync`) -//! - [`pull_sync`] — Background pull-sync dispatcher for organic routes //! - [`page_load`] — Single-use page-load write permit and row changes //! - [`write_plan`] — Plans a navigation's single row change //! - [`provider`] — Identity provider interface and configuration @@ -56,8 +55,6 @@ pub mod page_load; pub mod partner; pub mod prebid_eids; pub mod provider; -pub mod pull_sync; -pub(crate) mod pull_sync_marker; pub mod rate_limiter; pub mod registry; pub mod write_plan; @@ -99,7 +96,6 @@ use self::coordinator::results::LookupResults; use self::kv::KvIdentityGraph; use self::kv_types::KvEntry; use self::page_load::{AfterResponseWrite, NavigationPermit}; -use self::pull_sync_marker::{PullSyncMarkerState, validate_marker_state}; /// Bounded request classifications that may persist browser EID cookies. /// @@ -193,8 +189,8 @@ pub use generation::{ struct RequestEc { /// EC ID from the `ts-ec` cookie, if present. cookie_ec: Option, - /// Pull-sync completeness marker, if present. - pull_sync_marker: Option, + /// Whether the retired pull-sync completeness marker is present. + pull_marker_present: bool, /// The parsed cookie jar (retained for consent pipeline input). jar: Option, } @@ -211,15 +207,13 @@ fn parse_ec_from_request(req: &Request) -> Result>, - /// Browser-carried proof of recent pull-partner completeness. - pull_sync_marker: PullSyncMarkerState, + /// Whether the request carried the retired pull-sync marker cookie. + pull_marker_present: bool, /// Allowed returning-user EID persistence source, assigned only after request filters pass. eid_sync_source: Option, /// Navigation permit issued by the router for page-load navigations. @@ -404,7 +398,7 @@ impl EcContext { kv_snapshot: EcKvSnapshot::NotRead, recovery_eligible: false, client_eids: None, - pull_sync_marker: PullSyncMarkerState::from_cookie(parsed.pull_sync_marker), + pull_marker_present: parsed.pull_marker_present, eid_sync_source: None, navigation_permit: PermitSlot::default(), after_response_permit: AfterResponsePermitSlot::default(), @@ -476,7 +470,6 @@ impl EcContext { } self.ec_value = Some(ec_id); self.ec_generated = true; - self.pull_sync_marker.invalidate_for_replaced_ec(); Ok(()) } @@ -595,35 +588,16 @@ impl EcContext { self.client_eids.as_deref() } - /// Validates a browser completeness marker against the active EC and partner set. - pub(crate) fn validate_pull_sync_marker( - &mut self, - settings: &Settings, - registry: ®istry::PartnerRegistry, - ) { - validate_marker_state( - &mut self.pull_sync_marker, - settings, - registry, - self.ec_value.as_deref(), - ); - } - - /// Returns the current pull-sync marker state. + /// Whether the request carried the retired pull-sync marker cookie. #[must_use] - pub(crate) fn pull_sync_marker(&self) -> &PullSyncMarkerState { - &self.pull_sync_marker + pub(crate) fn pull_marker_present(&self) -> bool { + self.pull_marker_present } - /// Returns mutable pull-sync marker state for response reconciliation. - pub(crate) fn pull_sync_marker_mut(&mut self) -> &mut PullSyncMarkerState { - &mut self.pull_sync_marker - } - - /// Sets pull-sync marker state in focused unit tests. + /// Marks the retired pull-sync marker cookie as present, for tests. #[cfg(test)] - pub(crate) fn set_pull_sync_marker_for_test(&mut self, state: PullSyncMarkerState) { - self.pull_sync_marker = state; + pub(crate) fn set_pull_marker_present_for_test(&mut self, present: bool) { + self.pull_marker_present = present; } /// Stores the navigation permit issued by the router. @@ -678,7 +652,6 @@ impl EcContext { self.ec_value = Some(ec_id); self.ec_generated = true; self.kv_snapshot = snapshot; - self.pull_sync_marker.invalidate_for_replaced_ec(); } /// Returns whether EC creation is permitted by consent for this request. @@ -729,7 +702,7 @@ impl EcContext { kv_snapshot: EcKvSnapshot::NotRead, recovery_eligible: false, client_eids: None, - pull_sync_marker: PullSyncMarkerState::Absent, + pull_marker_present: false, eid_sync_source: None, navigation_permit: PermitSlot::default(), after_response_permit: AfterResponsePermitSlot::default(), @@ -758,7 +731,7 @@ impl EcContext { kv_snapshot: EcKvSnapshot::NotRead, recovery_eligible: false, client_eids: None, - pull_sync_marker: PullSyncMarkerState::Absent, + pull_marker_present: false, eid_sync_source: None, navigation_permit: PermitSlot::default(), after_response_permit: AfterResponsePermitSlot::default(), @@ -790,7 +763,7 @@ impl EcContext { kv_snapshot: EcKvSnapshot::NotRead, recovery_eligible: false, client_eids: None, - pull_sync_marker: PullSyncMarkerState::Absent, + pull_marker_present: false, eid_sync_source: None, navigation_permit: PermitSlot::default(), after_response_permit: AfterResponsePermitSlot::default(), @@ -1002,6 +975,36 @@ mod tests { assert_eq!(snapshot.generation_for("ec-1"), None); } + #[test] + fn leftover_pull_marker_is_expired() { + let settings = create_test_settings(); + let req = create_test_request(&[("cookie", "ts-ec-pull-complete=x")]); + let mut ec = EcContext::read_from_request(&settings, &req, &noop_services()) + .expect("should read EC context"); + let mut response = http::Response::new(EdgeBody::empty()); + + finalize::ec_finalize_response( + &settings, + &mut ec, + None, + ®istry::PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert!( + response + .headers() + .get_all(http::header::SET_COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .any(|cookie| cookie.starts_with("ts-ec-pull-complete=;") + && cookie.contains("Max-Age=0")), + "should expire the leftover marker" + ); + } + #[test] fn read_from_request_ignores_header_ec() { let settings = create_test_settings(); diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs deleted file mode 100644 index fc1f89a861..0000000000 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ /dev/null @@ -1,1247 +0,0 @@ -//! Pull sync background dispatch. -//! -//! Launches partner pull-sync requests for organic traffic after the client -//! response has been sent. Dispatch is best-effort and never affects client -//! response status. -//! -//! Pull sync currently fills missing partner UIDs only. Once a partner UID is -//! present in the EC identity entry, it is not periodically refreshed because -//! the entry no longer stores per-partner sync timestamps. - -use edgezero_core::body::Body as EdgeBody; -use http::{Method, StatusCode, header}; -use serde::Deserialize; -use url::Url; - -use crate::consent::allows_eid_persistence; -use crate::platform::{ - DEFAULT_FIRST_BYTE_TIMEOUT, PlatformBackendSpec, PlatformHttpRequest, PlatformPendingRequest, - PlatformResponse, RuntimeServices, -}; -use crate::settings::Settings; - -use super::generation::{ec_hash, is_valid_ec_id}; -use super::kv::{KvIdentityGraph, PartnerIdUpdate}; -use super::kv_types::KvEntry; -use super::rate_limiter::RateLimiter; -use super::registry::{PartnerConfig, PartnerRegistry}; - -// `current_timestamp` is defined in the parent `ec` module. -use super::EcContext; -use super::EcKvSnapshot; -use super::current_timestamp; - -/// Inputs needed to dispatch pull sync after response flush. -#[derive(Debug, Clone)] -pub struct PullSyncContext { - ec_id: String, - snapshot: EcKvSnapshot, -} - -impl PullSyncContext { - /// Returns the EC ID for the request. - #[must_use] - pub fn ec_id(&self) -> &str { - &self.ec_id - } -} - -struct InFlightPull { - source_domain: String, - pending: PlatformPendingRequest, -} - -#[derive(Debug, Deserialize)] -struct PullSyncResponse { - uid: Option, -} - -/// Builds post-send pull-sync context from the route EC context. -/// -/// Returns `None` when there are no pull-enabled partners, when consent denies -/// EC, when there is no valid active EC ID, or when the request snapshot holds -/// no consented row that is still missing at least one pull-partner UID. -#[must_use] -pub fn build_pull_sync_context( - ec_context: &EcContext, - registry: &PartnerRegistry, -) -> Option { - // Without EID-persistence consent (TCF Purpose 1 + 4), finalization - // withholds browser-supplied partner UIDs. That leaves partner slots empty, - // which pull sync would otherwise read as eligible and disclose the EC ID - // to partners for exactly the users who opted out. - if registry.pull_enabled_partners().is_empty() - || !ec_context.ec_allowed() - || !allows_eid_persistence(ec_context.consent()) - { - return None; - } - - let ec_id_ref = ec_context.ec_value()?; - if !is_valid_ec_id(ec_id_ref) { - log::debug!("Pull sync: skipping dispatch because active EC ID is invalid format"); - return None; - } - - let entry = ec_context.kv_snapshot().entry_for(ec_id_ref)?; - if !entry.consent.ok || entry_is_pull_complete(entry, registry) { - return None; - } - - let ec_id = ec_id_ref.to_owned(); - let snapshot = ec_context.kv_snapshot().clone(); - Some(PullSyncContext { ec_id, snapshot }) -} - -/// Dispatches partner pull-sync requests in the background. -/// -/// This function is best-effort: all errors are logged and swallowed. -/// -/// # Consent revalidation -/// -/// Pull sync runs *after* the page response has been flushed to the browser and -/// discloses the raw `ec_id` to partners, so it cannot be authorized by the -/// request snapshot alone: a concurrent CMP withdrawal can tombstone the row in -/// the window between snapshot capture and dispatch. The live row is re-read -/// immediately before the first partner request and anything short of a live, -/// consenting entry cancels the dispatch. The read is deferred until after the -/// snapshot-based eligibility filter, so a request with nothing to pull still -/// costs no KV operation, and the fresh snapshot is reused for the write-back -/// instead of the stale one. -/// -/// # Panics -/// -/// Panics if the HTTP request builder produces an invalid request, which -/// cannot happen with the hardcoded method and well-formed URI used here. -pub fn dispatch_pull_sync( - settings: &Settings, - kv: &KvIdentityGraph, - registry: &PartnerRegistry, - rate_limiter: &dyn RateLimiter, - context: &PullSyncContext, - services: &RuntimeServices, -) { - let now = current_timestamp(); - let mut pull_partners = registry.pull_enabled_partners(); - - if pull_partners.is_empty() { - return; - } - - let Some(request_entry) = context.snapshot.entry_for(context.ec_id()) else { - log::debug!( - "Pull sync: skipping dispatch for '{}' because the request captured no usable \ - identity snapshot", - super::log_id(context.ec_id()) - ); - return; - }; - if !request_entry.consent.ok { - return; - } - - // Sort by source domain for deterministic ordering, then apply a rotating - // hourly offset so that different partners get dispatch priority (§10.3). - pull_partners.sort_by(|a, b| a.source_domain.cmp(&b.source_domain)); - - log::debug!( - "Pull sync: {} pull-enabled partners after filtering", - pull_partners.len(), - ); - - // Rotate the partner list so that the starting partner changes each - // hour. This ensures fair distribution when max_concurrency limits - // how many partners are dispatched per request. - let offset = (now / 3600) as usize % pull_partners.len(); - pull_partners.rotate_left(offset); - - // Drop partners the request snapshot already has a UID for before paying - // for the revalidation read: a dispatch with nothing to pull must not add a - // KV operation. - pull_partners.retain(|partner| is_partner_pull_eligible(partner, Some(request_entry))); - if pull_partners.is_empty() { - return; - } - - let live_snapshot = kv.load_snapshot(context.ec_id()); - let Some(live_entry) = live_snapshot.entry_for(context.ec_id()) else { - log::warn!( - "Pull sync: skipping dispatch for '{}' because the live identity row could not be \ - confirmed", - super::log_id(context.ec_id()) - ); - return; - }; - if !live_entry.consent.ok { - log::info!( - "Pull sync: skipping dispatch for '{}' because consent was withdrawn after the \ - request snapshot was captured", - super::log_id(context.ec_id()) - ); - return; - } - - let max_concurrency = settings.ec.pull_sync_concurrency.max(1); - let mut in_flight: Vec = Vec::new(); - let mut updates = Vec::new(); - - for partner in pull_partners { - // Re-checked against the live row: a concurrent request may have filled - // this partner's UID since the request snapshot was captured. - if !is_partner_pull_eligible(partner, Some(live_entry)) { - continue; - } - - let Some(url) = validated_pull_sync_url(partner) else { - continue; - }; - - let rate_key = pull_rate_limit_key(&partner.source_domain, context.ec_id()); - match rate_limiter.exceeded(&rate_key, partner.pull_sync_rate_limit) { - Ok(true) => { - log::debug!( - "Pull sync: rate-limited partner '{}' for ec_id '{}'", - partner.source_domain, - super::log_id(context.ec_id()) - ); - continue; - } - Ok(false) => {} - Err(err) => { - log::warn!( - "Pull sync: failed to read rate limit for partner '{}': {err:?}", - partner.source_domain - ); - continue; - } - } - - let Some(token) = partner.ts_pull_token.as_ref() else { - log::warn!( - "Pull sync: partner '{}' enabled but missing ts_pull_token", - partner.source_domain - ); - continue; - }; - - let request_url = build_pull_request_url(url, context.ec_id()); - let scheme = request_url.scheme().to_string(); - let host = request_url.host_str().unwrap_or_default().to_string(); - let port = request_url.port(); - - let backend_name = match services.backend().ensure(&PlatformBackendSpec { - scheme, - host, - port, - host_header_override: None, - certificate_check: settings.proxy.certificate_check, - first_byte_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, - between_bytes_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, - discriminator: None, - }) { - Ok(name) => name, - Err(err) => { - log::warn!( - "Pull sync: failed to resolve backend for partner '{}': {err:?}", - partner.source_domain - ); - continue; - } - }; - - let request = http::Request::builder() - .method(Method::GET) - .uri(request_url.as_str()) - .header("authorization", format!("Bearer {}", token.expose())) - .body(EdgeBody::empty()) - .expect("should build pull sync request"); - - let pending = match futures::executor::block_on( - services - .http_client() - .send_async(PlatformHttpRequest::new(request, backend_name)), - ) { - Ok(pending) => pending, - Err(err) => { - log::warn!( - "Pull sync: failed to dispatch partner '{}': {err:?}", - partner.source_domain - ); - continue; - } - }; - - in_flight.push(InFlightPull { - source_domain: partner.source_domain.clone(), - pending, - }); - - if in_flight.len() >= max_concurrency { - drain_pull_batch(&mut in_flight, services, &mut updates); - } - } - - drain_pull_batch(&mut in_flight, services, &mut updates); - if !updates.is_empty() { - // Write back from the revalidated snapshot, not the request one: its - // generation is current, so the first CAS attempt is not spent losing a - // conflict against the read that authorized this dispatch. - let outcome = - kv.upsert_partner_ids_from_snapshot(context.ec_id(), &updates, live_snapshot.clone()); - if matches!(outcome, EcKvSnapshot::Failed { .. }) { - log::warn!( - "Pull sync: failed to persist partner updates for '{}'", - super::log_id(context.ec_id()) - ); - } - } -} - -/// Returns whether a consented live entry holds an ID for every pull-enabled -/// partner. Always returns `false` when there are no pull-enabled partners. -#[must_use] -pub(crate) fn entry_is_pull_complete(entry: &KvEntry, registry: &PartnerRegistry) -> bool { - let pull_partners = registry.pull_enabled_partners(); - !pull_partners.is_empty() - && entry.consent.ok - && pull_partners - .iter() - .all(|partner| !is_partner_pull_eligible(partner, Some(entry))) -} - -fn is_partner_pull_eligible(partner: &PartnerConfig, kv_entry: Option<&KvEntry>) -> bool { - kv_entry - .and_then(|entry| entry.ids.get(&partner.source_domain)) - .is_none() -} - -fn validated_pull_sync_url(partner: &PartnerConfig) -> Option { - let pull_sync_url = partner.pull_sync_url.as_deref()?; - let parsed = match Url::parse(pull_sync_url) { - Ok(url) => url, - Err(err) => { - log::error!( - "Pull sync: partner '{}' has invalid pull_sync_url '{}': {err}", - partner.source_domain, - pull_sync_url - ); - return None; - } - }; - - if parsed.scheme() != "https" { - log::error!( - "Pull sync: partner '{}' pull_sync_url must use HTTPS, got scheme '{}'", - partner.source_domain, - parsed.scheme() - ); - return None; - } - - let Some(hostname) = parsed.host_str() else { - log::error!( - "Pull sync: partner '{}' pull_sync_url has no hostname: {}", - partner.source_domain, - pull_sync_url - ); - return None; - }; - - let hostname = hostname.trim_end_matches('.').to_ascii_lowercase(); - if !partner.pull_sync_allowed_domains.iter().any(|domain| { - domain - .trim() - .trim_end_matches('.') - .eq_ignore_ascii_case(&hostname) - }) { - log::error!( - "Pull sync: partner '{}' URL host '{}' not in pull_sync_allowed_domains", - partner.source_domain, - hostname - ); - return None; - } - - Some(parsed) -} - -fn build_pull_request_url(mut base_url: Url, ec_id: &str) -> Url { - base_url.query_pairs_mut().append_pair("ec_id", ec_id); - base_url -} - -fn pull_rate_limit_key(source_domain: &str, ec_id: &str) -> String { - format!("pull:{source_domain}:{}", ec_hash(ec_id)) -} - -fn drain_pull_batch( - in_flight: &mut Vec, - services: &RuntimeServices, - updates: &mut Vec, -) { - for pending in in_flight.drain(..) { - let source_domain = pending.source_domain; - // All requests were dispatched up front via send_async, so waiting on - // each in turn does not change concurrency. - let response = - match futures::executor::block_on(services.http_client().wait(pending.pending)) { - Ok(response) => response, - Err(err) => { - log::warn!( - "Pull sync: request failed for partner '{}': {err:?}", - source_domain - ); - continue; - } - }; - - let Some(uid) = extract_pull_uid(response, &source_domain) else { - continue; - }; - - updates.push(PartnerIdUpdate::new(source_domain, uid)); - } -} - -/// Maximum response body size accepted from pull sync partners (64 KiB). -/// -/// The expected response is `{"uid":""}`, so 64 KiB is generous. -/// This prevents a misbehaving partner from exhausting WASM memory. -const MAX_PULL_RESPONSE_BYTES: usize = 64 * 1024; - -fn response_content_length_exceeds_limit(response: &PlatformResponse, source_domain: &str) -> bool { - let Some(value) = response.response.headers().get(header::CONTENT_LENGTH) else { - return false; - }; - - let Some(value) = value.to_str().ok() else { - log::warn!( - "Pull sync: partner '{}' returned invalid Content-Length header, rejecting", - source_domain - ); - return true; - }; - - let Ok(length) = value.parse::() else { - log::warn!( - "Pull sync: partner '{}' returned malformed Content-Length header, rejecting", - source_domain - ); - return true; - }; - - if length > MAX_PULL_RESPONSE_BYTES { - log::warn!( - "Pull sync: partner '{}' returned oversized Content-Length ({} bytes), rejecting", - source_domain, - length - ); - return true; - } - - false -} - -fn extract_pull_uid(response: PlatformResponse, source_domain: &str) -> Option { - let status = response.response.status(); - - if status == StatusCode::NOT_FOUND { - log::debug!( - "Pull sync: partner '{}' returned 404, treating as no-op", - source_domain - ); - return None; - } - - if !status.is_success() { - log::warn!( - "Pull sync: partner '{}' returned non-success status {}", - source_domain, - status - ); - return None; - } - - if response_content_length_exceeds_limit(&response, source_domain) { - return None; - } - - let body = response - .response - .into_body() - .into_bytes() - .unwrap_or_default(); - if body.len() > MAX_PULL_RESPONSE_BYTES { - log::warn!( - "Pull sync: partner '{}' returned oversized response ({} bytes), rejecting", - source_domain, - body.len() - ); - return None; - } - let payload = match serde_json::from_slice::(&body) { - Ok(payload) => payload, - Err(err) => { - log::warn!( - "Pull sync: partner '{}' returned invalid JSON body: {err}", - source_domain - ); - return None; - } - }; - - use super::kv_types::MAX_UID_LENGTH; - - let uid = payload.uid.filter(|value| !value.trim().is_empty()); - match uid { - None => { - log::debug!( - "Pull sync: partner '{}' returned null/empty uid, treating as no-op", - source_domain - ); - None - } - Some(ref value) if value.len() > MAX_UID_LENGTH => { - log::warn!( - "Pull sync: partner '{}' returned uid exceeding {} bytes (got {}), rejecting", - source_domain, - MAX_UID_LENGTH, - value.len() - ); - None - } - _ => uid, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::consent::types::ConsentContext; - use crate::ec::kv_types::KvEntry; - use crate::platform::PlatformResponse; - use crate::redacted::Redacted; - use crate::test_support::tests::purpose_one_only_consent; - - fn make_response(status: u16, body: &[u8]) -> PlatformResponse { - PlatformResponse::new( - edgezero_core::http::response_builder() - .status(status) - .body(EdgeBody::from(body.to_vec())) - .expect("should build test response"), - ) - } - - fn make_response_with_content_length( - status: u16, - content_length: usize, - body: &[u8], - ) -> PlatformResponse { - PlatformResponse::new( - edgezero_core::http::response_builder() - .status(status) - .header(header::CONTENT_LENGTH, content_length.to_string()) - .body(EdgeBody::from(body.to_vec())) - .expect("should build test response"), - ) - } - - fn pull_partner(ttl_sec: u64) -> PartnerConfig { - PartnerConfig { - name: "SSP X".to_owned(), - api_key_hash: Some("deadbeef".to_owned()), - bidstream_enabled: true, - source_domain: "ssp.example.com".to_owned(), - openrtb_atype: 3, - batch_rate_limit: 60, - pull_sync_enabled: true, - pull_sync_url: Some("https://sync.partner.test/pull".to_owned()), - pull_sync_allowed_domains: vec!["sync.partner.test".to_owned()], - pull_sync_ttl_sec: ttl_sec, - pull_sync_rate_limit: 20, - ts_pull_token: Some(Redacted::new("token".to_owned())), - provider_owned: false, - owner_tcf_vendor_ids: Vec::new(), - excluded_where_gdpr_applies: false, - } - } - - #[test] - fn build_pull_sync_context_returns_context_when_valid() { - let consent = ConsentContext { - jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, - ..ConsentContext::default() - }; - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let mut ec_context = EcContext::new_for_test(Some(ec_id.clone()), consent); - let graph = KvIdentityGraph::in_memory("pull_store"); - ec_context.set_kv_snapshot(seed_present_snapshot(&graph, &ec_id)); - let registry = PartnerRegistry::from_config(&[pull_enabled_ec_partner("ssp.example.com")]) - .expect("should build registry"); - - let context = build_pull_sync_context(&ec_context, ®istry) - .expect("should build pull sync context for valid EC"); - assert_eq!( - context.ec_id(), - ec_context.ec_value().expect("ec should be present"), - "should capture the EC ID from context" - ); - } - - #[test] - fn build_pull_sync_context_rejects_invalid_ec_id() { - let consent = ConsentContext { - jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, - ..ConsentContext::default() - }; - let ec_context = EcContext::new_for_test(Some("invalid-ec".to_owned()), consent); - let registry = PartnerRegistry::from_config(&[pull_enabled_ec_partner("ssp.example.com")]) - .expect("should build registry"); - - let context = build_pull_sync_context(&ec_context, ®istry); - assert!( - context.is_none(), - "should reject pull sync context when EC ID format is invalid" - ); - } - - #[test] - fn build_pull_sync_context_skips_when_eid_persistence_is_denied() { - // Purpose 1 granted keeps the EC, but Purpose 4 denied withholds EID - // writes and leaves partner slots empty. Pull sync must not read those - // empty slots as eligible and disclose the EC ID to partners. - let consent = purpose_one_only_consent(); - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let mut ec_context = EcContext::new_for_test(Some(ec_id.clone()), consent); - let graph = KvIdentityGraph::in_memory("pull_store"); - ec_context.set_kv_snapshot(seed_present_snapshot(&graph, &ec_id)); - let registry = PartnerRegistry::from_config(&[pull_enabled_ec_partner("ssp.example.com")]) - .expect("should build registry"); - assert!( - ec_context.ec_allowed(), - "should keep the EC allowed when Purpose 1 is granted" - ); - - let context = build_pull_sync_context(&ec_context, ®istry); - - assert!( - context.is_none(), - "should skip pull sync when TCF Purpose 4 is denied" - ); - } - - #[test] - fn build_pull_sync_context_skips_empty_registry_and_complete_snapshot() { - let consent = ConsentContext { - jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, - ..ConsentContext::default() - }; - let ec_id = format!("{}.ABC123", "a".repeat(64)); - let mut ec_context = EcContext::new_for_test(Some(ec_id.clone()), consent); - let graph = KvIdentityGraph::in_memory("pull_store"); - let mut snapshot = seed_present_snapshot(&graph, &ec_id); - let registry = PartnerRegistry::from_config(&[pull_enabled_ec_partner("ssp.example.com")]) - .expect("should build registry"); - - assert!( - build_pull_sync_context(&ec_context, &PartnerRegistry::empty()).is_none(), - "no pull partners should skip before graph construction" - ); - assert!( - build_pull_sync_context(&ec_context, ®istry).is_none(), - "an unread snapshot should skip before graph construction" - ); - - if let EcKvSnapshot::Present { entry, .. } = &mut snapshot { - entry.ids.insert( - "ssp.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId::single("uid".to_owned()), - ); - } - ec_context.set_kv_snapshot(snapshot); - assert!( - build_pull_sync_context(&ec_context, ®istry).is_none(), - "a complete snapshot should skip post-send work" - ); - } - - #[test] - fn partner_is_eligible_when_missing_from_entry() { - let partner = pull_partner(3600); - let entry = KvEntry::minimal("other_partner", "uid-1", 100); - - assert!( - is_partner_pull_eligible(&partner, Some(&entry)), - "should dispatch when partner has no stored UID" - ); - } - - #[test] - fn partner_is_not_eligible_when_already_present() { - let partner = pull_partner(3600); - let entry = KvEntry::minimal("ssp.example.com", "uid-1", 1000); - - assert!( - !is_partner_pull_eligible(&partner, Some(&entry)), - "should skip dispatch when partner already has a stored UID" - ); - } - - #[test] - fn completeness_requires_all_pull_partner_ids() { - let registry = PartnerRegistry::from_config(&[ - pull_enabled_ec_partner("a.example.com"), - pull_enabled_ec_partner("b.example.com"), - ]) - .expect("should build registry"); - let mut entry = KvEntry::minimal("a.example.com", "uid-a", 1_000); - - assert!( - !entry_is_pull_complete(&entry, ®istry), - "entry missing a pull partner ID should be incomplete" - ); - - entry.ids.insert( - "b.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId::single("uid-b".to_owned()), - ); - - assert!( - entry_is_pull_complete(&entry, ®istry), - "entry with every pull partner ID should be complete" - ); - } - - #[test] - fn validated_pull_sync_url_rejects_http_scheme() { - let mut partner = pull_partner(3600); - partner.pull_sync_url = Some("http://sync.partner.test/pull".to_owned()); - - let validated = validated_pull_sync_url(&partner); - assert!( - validated.is_none(), - "should reject pull_sync_url with HTTP scheme" - ); - } - - #[test] - fn validated_pull_sync_url_rejects_non_allowlisted_host() { - let mut partner = pull_partner(3600); - partner.pull_sync_url = Some("https://evil.test/pull".to_owned()); - - let validated = validated_pull_sync_url(&partner); - assert!( - validated.is_none(), - "should reject runtime pull_sync_url host outside allowlist" - ); - } - - #[test] - fn validated_pull_sync_url_accepts_normalized_allowlist_match() { - let mut partner = pull_partner(3600); - partner.pull_sync_url = Some("https://SYNC.PARTNER.TEST./pull".to_owned()); - partner.pull_sync_allowed_domains = vec!["sync.partner.test".to_owned()]; - - let validated = validated_pull_sync_url(&partner); - assert!( - validated.is_some(), - "should accept allowlist match after hostname normalization" - ); - } - - #[test] - fn build_pull_request_url_appends_ec_id() { - let url = Url::parse("https://sync.partner.test/pull?x=1").expect("should parse URL"); - let result = build_pull_request_url(url, "ecid123"); - - let query = result.query().expect("should have query string"); - assert!(query.contains("x=1"), "should preserve existing query"); - assert!(query.contains("ec_id=ecid123"), "should append ec_id"); - assert!( - !query.contains("ip="), - "should not forward client IP to partners" - ); - } - - #[test] - fn pull_rate_limit_key_uses_ec_hash_only() { - let first_ec_id = format!("{}.ABC123", "a".repeat(64)); - let second_ec_id = format!("{}.XYZ789", "a".repeat(64)); - - let first_key = pull_rate_limit_key("ssp.example.com", &first_ec_id); - let second_key = pull_rate_limit_key("ssp.example.com", &second_ec_id); - - assert_eq!( - first_key, second_key, - "should bucket different suffixes for the same EC hash together" - ); - assert_eq!( - first_key, - format!("pull:ssp.example.com:{}", "a".repeat(64)), - "should key pull-sync rate limiting by source domain and EC hash" - ); - } - - #[test] - fn extract_pull_uid_treats_404_as_noop() { - let response = make_response(404, b""); - - let uid = extract_pull_uid(response, "ssp.example.com"); - assert!(uid.is_none(), "should treat 404 as no-op"); - } - - #[test] - fn extract_pull_uid_treats_uid_null_as_noop() { - let response = make_response(200, b"{\"uid\":null}"); - - let uid = extract_pull_uid(response, "ssp.example.com"); - assert!(uid.is_none(), "should treat uid=null as no-op"); - } - - #[test] - fn extract_pull_uid_rejects_oversized_uid() { - let long_uid = "x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1); - let body = format!("{{\"uid\":\"{long_uid}\"}}"); - let response = make_response(200, body.as_bytes()); - - let uid = extract_pull_uid(response, "ssp.example.com"); - assert!(uid.is_none(), "should reject uid exceeding MAX_UID_LENGTH"); - } - - #[test] - fn extract_pull_uid_reads_uid_from_success_body() { - let response = make_response(200, b"{\"uid\":\"abc123\"}"); - - let uid = extract_pull_uid(response, "ssp.example.com"); - assert_eq!( - uid.as_deref(), - Some("abc123"), - "should parse uid from 200 body" - ); - } - - #[test] - fn extract_pull_uid_rejects_oversized_content_length_before_body_read() { - let response = make_response_with_content_length( - 200, - MAX_PULL_RESPONSE_BYTES + 1, - b"{\"uid\":\"abc123\"}", - ); - - let uid = extract_pull_uid(response, "ssp.example.com"); - assert!( - uid.is_none(), - "should reject oversized Content-Length before parsing body" - ); - } - - #[test] - fn extract_pull_uid_accepts_small_body_without_content_length() { - let response = make_response(200, b"{\"uid\":\"abc123\"}"); - - let uid = extract_pull_uid(response, "ssp.example.com"); - assert_eq!( - uid.as_deref(), - Some("abc123"), - "should accept small valid response without Content-Length" - ); - } - - #[test] - fn extract_pull_uid_rejects_body_larger_than_limit() { - let body = format!("{{\"uid\":\"{}\"}}", "x".repeat(MAX_PULL_RESPONSE_BYTES)); - let response = make_response(200, body.as_bytes()); - - let uid = extract_pull_uid(response, "ssp.example.com"); - assert!(uid.is_none(), "should reject body larger than limit"); - } - - #[test] - fn rotating_offset_distributes_partners_across_hours() { - // Simulate 3 partners sorted by source domain: alpha, beta, gamma. - let ids = vec!["alpha.example.com", "beta.example.com", "gamma.example.com"]; - - // Hour 0: offset = 0 % 3 = 0 → [alpha, beta, gamma] - let ts_h0: u64 = 100; // within hour 0 - let offset_h0 = (ts_h0 / 3600) as usize % ids.len(); - assert_eq!(offset_h0, 0, "hour 0 should start at index 0"); - - // Hour 1: offset = (3600 / 3600) % 3 = 1 → [beta, gamma, alpha] - let offset_h1 = (3600u64 / 3600) as usize % ids.len(); - assert_eq!(offset_h1, 1, "hour 1 should start at index 1"); - - // Hour 2: offset = (7200 / 3600) % 3 = 2 → [gamma, alpha, beta] - let offset_h2 = (7200u64 / 3600) as usize % ids.len(); - assert_eq!(offset_h2, 2, "hour 2 should start at index 2"); - - // Hour 3: offset = (10800 / 3600) % 3 = 0 → wraps back to [alpha, beta, gamma] - let offset_h3 = (10800u64 / 3600) as usize % ids.len(); - assert_eq!(offset_h3, 0, "hour 3 should wrap back to index 0"); - - // Verify rotate_left produces expected ordering - let mut rotated = ids.clone(); - rotated.rotate_left(offset_h1); - assert_eq!( - rotated, - vec!["beta.example.com", "gamma.example.com", "alpha.example.com"], - "hour 1 rotation should move beta to front" - ); - } - - // ----------------------------------------------------------------------- - // Snapshot-driven eligibility and request-wide aggregation - // ----------------------------------------------------------------------- - - use crate::ec::kv::TombstoneOutcome; - use crate::error::TrustedServerError; - use crate::platform::test_support::{StubHttpClient, build_services_with_http_client}; - use crate::settings::EcPartner; - use crate::test_support::tests::create_test_settings; - use error_stack::Report; - use std::sync::Arc; - - struct AllowAllRateLimiter; - - impl RateLimiter for AllowAllRateLimiter { - fn exceeded( - &self, - _key: &str, - _hourly_limit: u32, - ) -> Result> { - Ok(false) - } - } - - fn pull_enabled_ec_partner(source_domain: &str) -> EcPartner { - EcPartner { - name: format!("Partner {source_domain}"), - source_domain: source_domain.to_owned(), - openrtb_atype: EcPartner::default_openrtb_atype(), - bidstream_enabled: true, - api_token: Some(Redacted::new(format!( - "{source_domain}-api-token-32-bytes-minimum" - ))), - batch_rate_limit: EcPartner::default_batch_rate_limit(), - pull_sync_enabled: true, - pull_sync_url: Some(format!("https://{source_domain}/sync")), - pull_sync_allowed_domains: vec![source_domain.to_owned()], - pull_sync_ttl_sec: 3600, - pull_sync_rate_limit: 100, - ts_pull_token: Some(Redacted::new("outbound-token".to_owned())), - } - } - - fn snapshot_ec_id() -> String { - format!("{}.ABC123", "a".repeat(64)) - } - - fn seed_present_snapshot(graph: &KvIdentityGraph, ec_id: &str) -> EcKvSnapshot { - let mut entry = KvEntry::tombstone(1000); - entry.consent.ok = true; - graph.create(ec_id, &entry).expect("should seed live entry"); - graph.load_snapshot(ec_id) - } - - #[test] - fn dispatch_pull_sync_aggregates_batches_into_one_bulk_write() { - let mut settings = create_test_settings(); - // Force one partner per concurrency batch so responses span batches. - settings.ec.pull_sync_concurrency = 1; - let registry = PartnerRegistry::from_config(&[ - pull_enabled_ec_partner("alpha.example.com"), - pull_enabled_ec_partner("beta.example.com"), - ]) - .expect("should build pull registry"); - - let graph = KvIdentityGraph::in_memory("pull_store"); - let ec_id = snapshot_ec_id(); - let snapshot = seed_present_snapshot(&graph, &ec_id); - - let stub = Arc::new(StubHttpClient::new()); - // One JSON response per partner, drained across two concurrency batches. - stub.push_response(200, br#"{"uid":"synced-uid"}"#.to_vec()); - stub.push_response(200, br#"{"uid":"synced-uid"}"#.to_vec()); - let services = build_services_with_http_client(stub.clone()); - - let context = PullSyncContext { - ec_id: ec_id.clone(), - snapshot, - }; - dispatch_pull_sync( - &settings, - &graph, - ®istry, - &AllowAllRateLimiter, - &context, - &services, - ); - - let (entry, generation) = graph - .get(&ec_id) - .expect("should read store") - .expect("entry should exist"); - assert_eq!( - entry.ids.get("alpha.example.com").map(|id| id.uid.as_str()), - Some("synced-uid"), - "first partner UID should persist" - ); - assert_eq!( - entry.ids.get("beta.example.com").map(|id| id.uid.as_str()), - Some("synced-uid"), - "second partner UID should persist" - ); - assert_eq!( - generation, 2, - "two partner responses across batches must persist in exactly one bulk write" - ); - } - - #[test] - fn dispatch_pull_sync_calls_and_persists_only_missing_partner() { - let mut settings = create_test_settings(); - settings.ec.pull_sync_concurrency = 4; - let registry = PartnerRegistry::from_config(&[ - pull_enabled_ec_partner("alpha.example.com"), - pull_enabled_ec_partner("beta.example.com"), - ]) - .expect("should build pull registry"); - let graph = KvIdentityGraph::in_memory("pull_store"); - let ec_id = snapshot_ec_id(); - graph - .create( - &ec_id, - &KvEntry::minimal("alpha.example.com", "existing-alpha", 1_000), - ) - .expect("should seed partial entry"); - let snapshot = graph.load_snapshot(&ec_id); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, br#"{"uid":"new-beta"}"#.to_vec()); - let services = build_services_with_http_client(stub.clone()); - let context = PullSyncContext { - ec_id: ec_id.clone(), - snapshot, - }; - - dispatch_pull_sync( - &settings, - &graph, - ®istry, - &AllowAllRateLimiter, - &context, - &services, - ); - - assert_eq!( - stub.recorded_backend_names().len(), - 1, - "should call only the missing partner" - ); - let (entry, generation) = graph - .get(&ec_id) - .expect("should read store") - .expect("entry should exist"); - assert_eq!( - entry - .ids - .get("alpha.example.com") - .map(|partner_id| partner_id.uid.as_str()), - Some("existing-alpha"), - "existing UID should remain unchanged" - ); - assert_eq!( - entry - .ids - .get("beta.example.com") - .map(|partner_id| partner_id.uid.as_str()), - Some("new-beta"), - "missing UID should persist" - ); - assert_eq!(generation, 2, "missing UID should use one bulk CAS write"); - } - - #[test] - fn dispatch_pull_sync_skips_non_present_snapshots() { - let mut settings = create_test_settings(); - settings.ec.pull_sync_concurrency = 4; - let registry = - PartnerRegistry::from_config(&[pull_enabled_ec_partner("alpha.example.com")]) - .expect("should build registry"); - let graph = KvIdentityGraph::in_memory("pull_store"); - let ec_id = snapshot_ec_id(); - let stub = Arc::new(StubHttpClient::new()); - let services = build_services_with_http_client(stub.clone()); - - for snapshot in [ - EcKvSnapshot::NotRead, - EcKvSnapshot::Missing { - ec_id: ec_id.clone(), - }, - EcKvSnapshot::Failed { - ec_id: ec_id.clone(), - }, - ] { - let context = PullSyncContext { - ec_id: ec_id.clone(), - snapshot, - }; - dispatch_pull_sync( - &settings, - &graph, - ®istry, - &AllowAllRateLimiter, - &context, - &services, - ); - } - - assert!( - stub.recorded_backend_names().is_empty(), - "not-read, missing, and failed snapshots must not dispatch pull sync" - ); - assert!( - graph.get(&ec_id).expect("should read store").is_none(), - "no snapshot state should create a missing root" - ); - } - - #[test] - fn dispatch_pull_sync_skips_tombstone_snapshot() { - let mut settings = create_test_settings(); - settings.ec.pull_sync_concurrency = 4; - let registry = - PartnerRegistry::from_config(&[pull_enabled_ec_partner("alpha.example.com")]) - .expect("should build registry"); - let graph = KvIdentityGraph::in_memory("pull_store"); - let ec_id = snapshot_ec_id(); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(KvEntry::tombstone(1000)), - generation: Some(1), - }; - let stub = Arc::new(StubHttpClient::new()); - let services = build_services_with_http_client(stub.clone()); - - let context = PullSyncContext { - ec_id: ec_id.clone(), - snapshot, - }; - dispatch_pull_sync( - &settings, - &graph, - ®istry, - &AllowAllRateLimiter, - &context, - &services, - ); - - assert!( - stub.recorded_backend_names().is_empty(), - "a tombstone snapshot must not dispatch pull sync" - ); - } - #[test] - fn dispatch_pull_sync_skips_dispatch_when_ec_is_tombstoned_after_snapshot_capture() { - // The request snapshot authorized pull sync, then a concurrent request - // completed a CMP withdrawal while the page response was in flight. - // Pull sync runs post-send and discloses the raw `ec_id` to partners, so - // it must revalidate the live row and cancel rather than leak an - // identity the user just withdrew. - let mut settings = create_test_settings(); - settings.ec.pull_sync_concurrency = 4; - let registry = - PartnerRegistry::from_config(&[pull_enabled_ec_partner("alpha.example.com")]) - .expect("should build registry"); - let graph = KvIdentityGraph::in_memory("pull_store"); - let ec_id = snapshot_ec_id(); - let snapshot = seed_present_snapshot(&graph, &ec_id); - - // Concurrent withdrawal lands after the snapshot was captured. - assert_eq!( - graph - .write_withdrawal_tombstone(&ec_id, drop) - .expect("should tombstone the row"), - TombstoneOutcome::Written, - "should tombstone the row the snapshot still reports as present" - ); - - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, br#"{"uid":"leaked-uid"}"#.to_vec()); - let services = build_services_with_http_client(stub.clone()); - - let context = PullSyncContext { - ec_id: ec_id.clone(), - snapshot, - }; - dispatch_pull_sync( - &settings, - &graph, - ®istry, - &AllowAllRateLimiter, - &context, - &services, - ); - - assert!( - stub.recorded_backend_names().is_empty(), - "a withdrawal that lands after snapshot capture must cancel post-send pull sync" - ); - let (entry, _) = graph - .get(&ec_id) - .expect("should read store") - .expect("should keep the tombstone"); - assert!( - entry.ids.is_empty(), - "no partner UID may be written back onto a tombstoned row" - ); - } - - #[test] - fn dispatch_pull_sync_skips_revalidation_read_when_no_partner_is_eligible() { - // Every pull-enabled partner already has a UID in the request snapshot, - // so there is nothing to dispatch. The revalidation read exists to - // authorize outbound calls; with no calls to authorize it must not cost - // a KV operation. - let mut settings = create_test_settings(); - settings.ec.pull_sync_concurrency = 4; - let registry = - PartnerRegistry::from_config(&[pull_enabled_ec_partner("alpha.example.com")]) - .expect("should build registry"); - let graph = KvIdentityGraph::in_memory("pull_store"); - let ec_id = snapshot_ec_id(); - let mut entry = KvEntry::tombstone(1000); - entry.consent.ok = true; - entry.ids.insert( - "alpha.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId::single("already-known".to_owned()), - ); - graph - .create(&ec_id, &entry) - .expect("should seed live entry"); - let snapshot = EcKvSnapshot::Present { - ec_id: ec_id.clone(), - entry: Box::new(entry), - generation: Some(1), - }; - - let stub = Arc::new(StubHttpClient::new()); - let services = build_services_with_http_client(stub.clone()); - - let context = PullSyncContext { - ec_id: ec_id.clone(), - snapshot, - }; - dispatch_pull_sync( - &settings, - &graph, - ®istry, - &AllowAllRateLimiter, - &context, - &services, - ); - - assert!( - stub.recorded_backend_names().is_empty(), - "a fully synced entry must not dispatch pull sync" - ); - } -} diff --git a/crates/trusted-server-core/src/ec/pull_sync_marker.rs b/crates/trusted-server-core/src/ec/pull_sync_marker.rs deleted file mode 100644 index 7cc86898b2..0000000000 --- a/crates/trusted-server-core/src/ec/pull_sync_marker.rs +++ /dev/null @@ -1,689 +0,0 @@ -//! Short-lived browser marker for complete pull-sync partner state. - -use core::fmt; - -use edgezero_core::body::Body as EdgeBody; -use hmac::{Hmac, KeyInit as _, Mac as _}; -use http::{HeaderValue, Response, header}; -use sha2::{Digest as _, Sha256}; - -use crate::constants::COOKIE_TS_EC_PULL_COMPLETE; -use crate::redacted::Redacted; -use crate::settings::Settings; - -use super::pull_sync::entry_is_pull_complete; -use super::registry::PartnerRegistry; -use super::{EcKvSnapshot, current_timestamp}; - -type HmacSha256 = Hmac; - -const MARKER_VERSION: &str = "v1"; -const MARKER_KEY_LABEL: &[u8] = b"trusted-server/ec-pull-complete/key/v1"; -const MARKER_MAX_AGE_SECS: u64 = 60 * 60; -const MARKER_CLOCK_SKEW_SECS: u64 = 60; -const MAX_MARKER_LENGTH: usize = 256; - -/// Request-local validation state for the pull-sync completeness marker. -#[derive(Clone, Default)] -pub(crate) enum PullSyncMarkerState { - /// No marker cookie was present. - #[default] - Absent, - /// A marker was present but has not been checked against the active EC and partner set. - Unvalidated(Redacted), - /// A present marker failed validation or was disproved by authoritative KV state. - Invalid, - /// The marker is valid until the given Unix timestamp. - Valid { expires_at: u64 }, -} - -impl fmt::Debug for PullSyncMarkerState { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Absent => formatter.write_str("Absent"), - Self::Unvalidated(_) => formatter.write_str("Unvalidated()"), - Self::Invalid => formatter.write_str("Invalid"), - Self::Valid { expires_at } => formatter - .debug_struct("Valid") - .field("expires_at", expires_at) - .finish(), - } - } -} - -impl PullSyncMarkerState { - /// Creates marker state from an optional request cookie value. - #[must_use] - pub(crate) fn from_cookie(value: Option) -> Self { - value - .map(Redacted::new) - .map_or(Self::Absent, Self::Unvalidated) - } - - /// Returns whether a marker cookie was present on the request. - #[must_use] - pub(crate) fn was_present(&self) -> bool { - !matches!(self, Self::Absent) - } - - /// Returns whether the marker is currently valid. - #[must_use] - pub(crate) fn is_valid(&self) -> bool { - matches!(self, Self::Valid { .. }) - } - - /// Invalidates state bound to a replaced active EC ID. - pub(crate) fn invalidate_for_replaced_ec(&mut self) { - if self.was_present() { - *self = Self::Invalid; - } - } -} - -/// Validates an unvalidated marker against the active EC and current pull-partner set. -pub(crate) fn validate_marker_state( - state: &mut PullSyncMarkerState, - settings: &Settings, - registry: &PartnerRegistry, - ec_id: Option<&str>, -) { - let PullSyncMarkerState::Unvalidated(value) = state else { - return; - }; - let valid_until = ec_id.and_then(|ec_id| { - validate_marker( - value.expose(), - settings, - registry, - ec_id, - current_timestamp(), - ) - }); - *state = valid_until.map_or(PullSyncMarkerState::Invalid, |expires_at| { - PullSyncMarkerState::Valid { expires_at } - }); -} - -/// Reconciles browser marker state against finalized authoritative KV state. -pub(crate) fn reconcile_marker( - settings: &Settings, - registry: &PartnerRegistry, - ec_id: Option<&str>, - snapshot: &EcKvSnapshot, - state: &mut PullSyncMarkerState, - response: &mut Response, -) { - let pull_partners = sorted_pull_partner_domains(registry); - if pull_partners.is_empty() { - expire_if_present(state, response); - return; - } - - let Some(ec_id) = ec_id else { - expire_if_present(state, response); - return; - }; - - if !matches!(snapshot, EcKvSnapshot::NotRead) && !snapshot.belongs_to(ec_id) { - expire_if_present(state, response); - return; - } - - match snapshot { - EcKvSnapshot::Present { .. } => { - let entry = snapshot - .entry_for(ec_id) - .expect("snapshot binding should be checked before marker reconciliation"); - if !entry.consent.ok { - expire_if_present(state, response); - } else if entry_is_pull_complete(entry, registry) { - if !state.is_valid() { - set_marker(settings, registry, ec_id, state, response); - } - } else { - expire_if_present(state, response); - } - } - EcKvSnapshot::Missing { .. } => { - expire_if_present(state, response); - } - EcKvSnapshot::Failed { .. } | EcKvSnapshot::NotRead => { - if matches!(state, PullSyncMarkerState::Invalid) { - expire_if_present(state, response); - } - } - } -} - -/// Expires the completeness marker regardless of KV state. -pub(crate) fn expire_marker(state: &mut PullSyncMarkerState, response: &mut Response) { - append_cookie(response, &format_marker_cookie("", 0)); - *state = PullSyncMarkerState::Absent; -} - -fn set_marker( - settings: &Settings, - registry: &PartnerRegistry, - ec_id: &str, - state: &mut PullSyncMarkerState, - response: &mut Response, -) { - let now = current_timestamp(); - let expires_at = now.saturating_add(MARKER_MAX_AGE_SECS); - let Some(value) = create_marker(settings, registry, ec_id, expires_at) else { - return; - }; - append_cookie(response, &format_marker_cookie(&value, MARKER_MAX_AGE_SECS)); - *state = PullSyncMarkerState::Valid { expires_at }; -} - -fn expire_if_present(state: &mut PullSyncMarkerState, response: &mut Response) { - if state.was_present() { - expire_marker(state, response); - } -} - -fn append_cookie(response: &mut Response, value: &str) { - match HeaderValue::from_str(value) { - Ok(value) => { - response.headers_mut().append(header::SET_COOKIE, value); - } - Err(err) => { - log::warn!("Skipping pull-sync marker cookie: invalid header value: {err}"); - } - } -} - -fn format_marker_cookie(value: &str, max_age: u64) -> String { - // Keep this cookie host-only. The signed payload does not include the - // request host, so a Domain attribute would allow replay across sibling - // hosts that share the domain-wide EC cookie and passphrase. - format!( - "{COOKIE_TS_EC_PULL_COMPLETE}={value}; Path=/; Secure; SameSite=Lax; Max-Age={max_age}; HttpOnly" - ) -} - -#[cfg(test)] -pub(crate) fn create_marker_for_test( - settings: &Settings, - registry: &PartnerRegistry, - ec_id: &str, -) -> String { - create_marker_for_test_with_expiry( - settings, - registry, - ec_id, - current_timestamp().saturating_add(MARKER_MAX_AGE_SECS), - ) -} - -#[cfg(test)] -pub(crate) fn create_marker_for_test_with_expiry( - settings: &Settings, - registry: &PartnerRegistry, - ec_id: &str, - expires_at: u64, -) -> String { - create_marker(settings, registry, ec_id, expires_at) - .expect("should create marker for non-empty test registry") -} - -fn create_marker( - settings: &Settings, - registry: &PartnerRegistry, - ec_id: &str, - expires_at: u64, -) -> Option { - let fingerprint = partner_set_fingerprint(registry)?; - let payload = marker_payload(ec_id, expires_at, &fingerprint); - let key = marker_key(settings); - let mut mac = HmacSha256::new_from_slice(&key).expect("should create marker HMAC"); - mac.update(payload.as_bytes()); - let tag = hex::encode(mac.finalize().into_bytes()); - Some(format!("{MARKER_VERSION}.{expires_at}.{fingerprint}.{tag}")) -} - -fn validate_marker( - value: &str, - settings: &Settings, - registry: &PartnerRegistry, - ec_id: &str, - now: u64, -) -> Option { - if value.len() > MAX_MARKER_LENGTH { - return None; - } - - let mut segments = value.split('.'); - let version = segments.next()?; - let expires = segments.next()?; - let fingerprint = segments.next()?; - let tag = segments.next()?; - if segments.next().is_some() || version != MARKER_VERSION { - return None; - } - - let expires_at = expires.parse::().ok()?; - let max_future = now - .saturating_add(MARKER_MAX_AGE_SECS) - .saturating_add(MARKER_CLOCK_SKEW_SECS); - if expires_at <= now || expires_at > max_future { - return None; - } - - let expected_fingerprint = partner_set_fingerprint(registry)?; - if fingerprint != expected_fingerprint { - return None; - } - - let tag = hex::decode(tag).ok()?; - if tag.len() != 32 { - return None; - } - - let payload = marker_payload(ec_id, expires_at, fingerprint); - let key = marker_key(settings); - let mut mac = HmacSha256::new_from_slice(&key).expect("should create marker HMAC"); - mac.update(payload.as_bytes()); - mac.verify_slice(&tag).ok()?; - Some(expires_at) -} - -fn marker_key(settings: &Settings) -> [u8; 32] { - let mut mac = HmacSha256::new_from_slice(settings.ec.passphrase.expose().as_bytes()) - .expect("should create marker key HMAC"); - mac.update(MARKER_KEY_LABEL); - mac.finalize().into_bytes().into() -} - -fn marker_payload(ec_id: &str, expires_at: u64, fingerprint: &str) -> String { - format!("{MARKER_VERSION}\0{ec_id}\0{expires_at}\0{fingerprint}") -} - -fn partner_set_fingerprint(registry: &PartnerRegistry) -> Option { - let domains = sorted_pull_partner_domains(registry); - if domains.is_empty() { - return None; - } - - let mut hasher = Sha256::new(); - hasher.update(b"trusted-server/ec-pull-partners/v1\0"); - for domain in domains { - hasher.update((domain.len() as u64).to_be_bytes()); - hasher.update(domain.as_bytes()); - } - Some(hex::encode(hasher.finalize())) -} - -fn sorted_pull_partner_domains(registry: &PartnerRegistry) -> Vec { - let mut domains = registry - .pull_enabled_partners() - .into_iter() - .map(|partner| partner.source_domain.clone()) - .collect::>(); - domains.sort(); - domains -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::ec::kv_types::KvEntry; - use crate::redacted::Redacted; - use crate::settings::{EcPartner, Settings}; - use crate::test_support::tests::create_test_settings; - - const EC_ID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; - - fn pull_partner(source_domain: &str) -> EcPartner { - EcPartner { - name: format!("Partner {source_domain}"), - source_domain: source_domain.to_owned(), - openrtb_atype: EcPartner::default_openrtb_atype(), - bidstream_enabled: true, - api_token: Some(Redacted::new(format!( - "token-{source_domain}-32-bytes-minimum-value" - ))), - batch_rate_limit: EcPartner::default_batch_rate_limit(), - pull_sync_enabled: true, - pull_sync_url: Some(format!("https://sync.{source_domain}/pull")), - pull_sync_allowed_domains: vec![format!("sync.{source_domain}")], - pull_sync_ttl_sec: EcPartner::default_pull_sync_ttl_sec(), - pull_sync_rate_limit: EcPartner::default_pull_sync_rate_limit(), - ts_pull_token: Some(Redacted::new("pull-token".to_owned())), - } - } - - fn settings_and_registry(domains: &[&str]) -> (Settings, PartnerRegistry) { - let mut settings = create_test_settings(); - settings.ec.partners = domains.iter().map(|domain| pull_partner(domain)).collect(); - let registry = - PartnerRegistry::from_ec(&settings.ec).expect("should build pull partner registry"); - (settings, registry) - } - - fn empty_response() -> Response { - Response::builder() - .status(200) - .body(EdgeBody::empty()) - .expect("should build response") - } - - fn live_snapshot(ec_id: &str, domains: &[&str]) -> EcKvSnapshot { - let mut entry = KvEntry::tombstone(1_000); - entry.consent.ok = true; - for domain in domains { - entry.ids.insert( - (*domain).to_owned(), - crate::ec::kv_types::KvPartnerId::single(format!("uid-{domain}")), - ); - } - EcKvSnapshot::Present { - ec_id: ec_id.to_owned(), - entry: Box::new(entry), - generation: Some(1), - } - } - - fn marker_cookies(response: &Response) -> Vec<&str> { - response - .headers() - .get_all(header::SET_COOKIE) - .iter() - .filter_map(|value| value.to_str().ok()) - .collect() - } - - fn assert_expired(state: &PullSyncMarkerState, response: &Response) { - assert!(matches!(state, PullSyncMarkerState::Absent)); - assert_eq!( - marker_cookies(response), - vec!["ts-ec-pull-complete=; Path=/; Secure; SameSite=Lax; Max-Age=0; HttpOnly"], - "should emit the exact host-only expiration cookie" - ); - } - - #[test] - fn marker_round_trip_binds_ec_and_partner_set() { - let (settings, registry) = settings_and_registry(&["a.example.com", "b.example.com"]); - let now = 1_000; - let marker = - create_marker(&settings, ®istry, EC_ID, now + 3_600).expect("should create marker"); - - assert_eq!( - validate_marker(&marker, &settings, ®istry, EC_ID, now), - Some(now + 3_600), - "should validate the issued marker" - ); - assert!( - validate_marker(&marker, &settings, ®istry, "wrong", now).is_none(), - "should reject a marker for another EC" - ); - } - - #[test] - fn marker_fingerprint_is_order_independent_and_set_sensitive() { - let (settings, first) = settings_and_registry(&["a.example.com", "b.example.com"]); - let (_, reordered) = settings_and_registry(&["b.example.com", "a.example.com"]); - let (_, changed) = settings_and_registry(&["a.example.com", "c.example.com"]); - let marker = create_marker(&settings, &first, EC_ID, 4_600).expect("should create marker"); - - assert!( - validate_marker(&marker, &settings, &reordered, EC_ID, 1_000).is_some(), - "config ordering should not change the marker" - ); - assert!( - validate_marker(&marker, &settings, &changed, EC_ID, 1_000).is_none(), - "partner-set changes should invalidate the marker" - ); - } - - #[test] - fn fingerprint_changes_for_enable_disable_add_and_remove() { - let (_, enabled_one) = settings_and_registry(&["a.example.com"]); - let (_, enabled_two) = settings_and_registry(&["a.example.com", "b.example.com"]); - let mut disabled_config = pull_partner("a.example.com"); - disabled_config.pull_sync_enabled = false; - let disabled = PartnerRegistry::from_config(&[disabled_config]) - .expect("should build disabled registry"); - let removed = PartnerRegistry::empty(); - - let base = partner_set_fingerprint(&enabled_one); - assert_ne!( - base, - partner_set_fingerprint(&enabled_two), - "adding a pull partner should change the fingerprint" - ); - assert_ne!( - base, - partner_set_fingerprint(&disabled), - "disabling a pull partner should change the fingerprint" - ); - assert_ne!( - base, - partner_set_fingerprint(&removed), - "removing a pull partner should change the fingerprint" - ); - assert_ne!( - partner_set_fingerprint(&disabled), - partner_set_fingerprint(&enabled_one), - "enabling a partner should change the fingerprint" - ); - } - - #[test] - fn reconcile_rejects_snapshot_bound_to_different_ec() { - let (settings, registry) = settings_and_registry(&["a.example.com"]); - let snapshot = live_snapshot("different-ec", &["a.example.com"]); - let mut valid_state = PullSyncMarkerState::Valid { expires_at: 4_600 }; - let mut valid_response = empty_response(); - - reconcile_marker( - &settings, - ®istry, - Some(EC_ID), - &snapshot, - &mut valid_state, - &mut valid_response, - ); - assert_expired(&valid_state, &valid_response); - - let mut absent_state = PullSyncMarkerState::Absent; - let mut absent_response = empty_response(); - reconcile_marker( - &settings, - ®istry, - Some(EC_ID), - &snapshot, - &mut absent_state, - &mut absent_response, - ); - assert!(marker_cookies(&absent_response).is_empty()); - assert!(matches!(absent_state, PullSyncMarkerState::Absent)); - } - - #[test] - fn authoritative_incomplete_states_expire_valid_marker() { - let (settings, registry) = settings_and_registry(&["a.example.com"]); - let mut tombstone = KvEntry::tombstone(1_000); - tombstone.ids.insert( - "a.example.com".to_owned(), - crate::ec::kv_types::KvPartnerId::single("stale".to_owned()), - ); - let snapshots = [ - live_snapshot(EC_ID, &[]), - EcKvSnapshot::Missing { - ec_id: EC_ID.to_owned(), - }, - EcKvSnapshot::Present { - ec_id: EC_ID.to_owned(), - entry: Box::new(tombstone), - generation: Some(1), - }, - ]; - - for snapshot in snapshots { - let mut state = PullSyncMarkerState::Valid { expires_at: 4_600 }; - let mut response = empty_response(); - reconcile_marker( - &settings, - ®istry, - Some(EC_ID), - &snapshot, - &mut state, - &mut response, - ); - assert_expired(&state, &response); - } - } - - #[test] - fn non_authoritative_states_preserve_valid_fixed_expiry_marker() { - let (settings, registry) = settings_and_registry(&["a.example.com"]); - let snapshots = [ - EcKvSnapshot::Failed { - ec_id: EC_ID.to_owned(), - }, - EcKvSnapshot::NotRead, - ]; - - for snapshot in snapshots { - let mut state = PullSyncMarkerState::Valid { expires_at: 4_600 }; - let mut response = empty_response(); - reconcile_marker( - &settings, - ®istry, - Some(EC_ID), - &snapshot, - &mut state, - &mut response, - ); - assert!(matches!( - state, - PullSyncMarkerState::Valid { expires_at: 4_600 } - )); - assert!(marker_cookies(&response).is_empty()); - } - } - - #[test] - fn steady_states_leave_the_marker_cookie_untouched() { - let (settings, registry) = settings_and_registry(&["a.example.com"]); - let complete = live_snapshot(EC_ID, &["a.example.com"]); - let incomplete = live_snapshot(EC_ID, &[]); - let cases = [ - (PullSyncMarkerState::Valid { expires_at: 4_600 }, &complete), - (PullSyncMarkerState::Absent, &incomplete), - ]; - - for (mut state, snapshot) in cases { - let mut response = empty_response(); - reconcile_marker( - &settings, - ®istry, - Some(EC_ID), - snapshot, - &mut state, - &mut response, - ); - assert!( - marker_cookies(&response).is_empty(), - "a steady marker state should not add a Set-Cookie header" - ); - } - } - - #[test] - fn invalid_marker_is_cleared_without_authoritative_snapshot() { - let (settings, registry) = settings_and_registry(&["a.example.com"]); - let mut state = PullSyncMarkerState::Invalid; - let mut response = empty_response(); - - reconcile_marker( - &settings, - ®istry, - Some(EC_ID), - &EcKvSnapshot::NotRead, - &mut state, - &mut response, - ); - - assert_expired(&state, &response); - } - - #[test] - fn marker_allows_sixty_seconds_of_issuer_clock_skew() { - let (settings, registry) = settings_and_registry(&["a.example.com"]); - let marker = - create_marker(&settings, ®istry, EC_ID, 4_600).expect("should create marker"); - - assert_eq!( - validate_marker(&marker, &settings, ®istry, EC_ID, 940), - Some(4_600), - "a marker from an issuer sixty seconds ahead should remain valid" - ); - } - - #[test] - fn marker_rejects_expired_overlong_and_tampered_values() { - let (settings, registry) = settings_and_registry(&["a.example.com"]); - let marker = - create_marker(&settings, ®istry, EC_ID, 4_600).expect("should create marker"); - let mut tampered = marker.clone(); - tampered.push('0'); - - assert!( - validate_marker(&marker, &settings, ®istry, EC_ID, 4_600).is_none(), - "expired marker should fail" - ); - assert!( - validate_marker(&marker, &settings, ®istry, EC_ID, 939).is_none(), - "marker beyond the lifetime and clock-skew allowance should fail" - ); - assert!( - validate_marker(&tampered, &settings, ®istry, EC_ID, 1_000).is_none(), - "tampering should fail" - ); - assert!( - validate_marker( - &"x".repeat(MAX_MARKER_LENGTH + 1), - &settings, - ®istry, - EC_ID, - 1_000 - ) - .is_none(), - "overlong marker should fail" - ); - } - - #[test] - fn marker_rejects_wrong_passphrase_and_empty_partner_set() { - let (settings, registry) = settings_and_registry(&["a.example.com"]); - let marker = - create_marker(&settings, ®istry, EC_ID, 4_600).expect("should create marker"); - let mut changed_settings = settings.clone(); - changed_settings.ec.passphrase = - Redacted::new("different-secret-key-32-bytes-minimum".to_owned()); - let empty = PartnerRegistry::empty(); - - assert!( - validate_marker(&marker, &changed_settings, ®istry, EC_ID, 1_000).is_none(), - "passphrase rotation should invalidate the marker" - ); - assert!( - create_marker(&settings, &empty, EC_ID, 4_600).is_none(), - "empty partner sets should not produce a marker" - ); - } - - #[test] - fn marker_cookie_is_host_only_and_secure() { - let cookie = format_marker_cookie("value", MARKER_MAX_AGE_SECS); - assert_eq!( - cookie, - "ts-ec-pull-complete=value; Path=/; Secure; SameSite=Lax; Max-Age=3600; HttpOnly" - ); - assert!(!cookie.contains("Domain="), "marker should be host-only"); - } -} diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index fe20f83122..4174c03748 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -17,6 +17,9 @@ use crate::settings::{Ec, EcPartner}; use super::partner::{hash_api_key, normalize_partner_source_domain}; +/// Provider kind under which pull-sync partners run as identity providers. +pub const PULL_SYNC_KIND: &str = "pull-sync"; + /// Minimum length for inbound partner Bearer API tokens. pub const MIN_API_TOKEN_LENGTH: usize = 32; @@ -252,8 +255,8 @@ impl PartnerRegistry { } /// Builds the registry from `[ec]` settings and marks every source claimed - /// by an enabled identity provider whose kind is registered as - /// provider-owned. + /// by an enabled identity provider, and every pull-sync partner when the + /// `pull-sync` kind is registered, as provider-owned. /// /// # Errors /// @@ -287,6 +290,12 @@ impl PartnerRegistry { registry.keep_vendor_gate(source, provider.tcf_vendor_id); } } + let pull_sync_registered = is_registered_kind(PULL_SYNC_KIND); + for partner in ec.partners.iter().filter(|partner| { + pull_sync_registered && partner.pull_sync_enabled && partner.pull_sync_url.is_some() + }) { + registry.mark_provider_owned(&partner.source_domain, None); + } Ok(registry) } @@ -373,16 +382,12 @@ impl PartnerRegistry { .is_some_and(|partner| partner.provider_owned) } - /// Returns all partners with `pull_sync_enabled = true` whose source no - /// identity provider owns. - /// - /// A provider writes full-fidelity records for the sources it owns, so a - /// pull-sync write there would overwrite them with a single UID. + /// Returns all partners with `pull_sync_enabled = true`. #[must_use] pub fn pull_enabled_partners(&self) -> Vec<&PartnerConfig> { self.by_source_domain .values() - .filter(|p| p.pull_sync_enabled && !p.provider_owned) + .filter(|p| p.pull_sync_enabled) .collect() } @@ -584,9 +589,15 @@ mod tests { #[test] fn from_ec_marks_enabled_provider_sources_only() { let mut ec = crate::test_support::tests::create_test_settings().ec; + let mut pull = make_partner("pull.example.com", &valid_api_token("pull-partner")); + pull.pull_sync_enabled = true; + pull.pull_sync_url = Some("https://sync.example.com/pull".to_owned()); + pull.pull_sync_allowed_domains = vec!["sync.example.com".to_owned()]; + pull.ts_pull_token = Some(Redacted::new("example-pull-token".to_owned())); ec.partners = vec![ make_partner("ids.example.com", &valid_api_token("ids-partner")), make_partner("other.example.com", &valid_api_token("other-partner")), + pull, ]; ec.identity_providers = vec![ toml::from_str( @@ -599,8 +610,9 @@ mod tests { .expect("should parse disabled provider"), ]; - let registry = PartnerRegistry::from_ec_with_kinds(&ec, |kind| kind == "k") - .expect("should build registry"); + let registry = + PartnerRegistry::from_ec_with_kinds(&ec, |kind| kind == "k" || kind == PULL_SYNC_KIND) + .expect("should build registry"); assert!( registry.is_provider_owned("ids.example.com"), @@ -617,6 +629,10 @@ mod tests { Some(vec![NonZeroU16::new(42).expect("should be non-zero")]), "should keep the disabled provider's vendor consent gate" ); + assert!( + registry.is_provider_owned("pull.example.com"), + "should mark pull-sync partner source" + ); } #[test] @@ -842,25 +858,6 @@ mod tests { assert!(result.is_err(), "should reject invalid source domain"); } - #[test] - fn pull_enabled_partners_excludes_provider_owned_sources() { - let mut pull_partner = make_partner("pull.example.com", &valid_api_token("token-p")); - pull_partner.api_token = None; - pull_partner.pull_sync_enabled = true; - pull_partner.pull_sync_url = Some("https://pull.example.com/sync".to_owned()); - pull_partner.pull_sync_allowed_domains = vec!["pull.example.com".to_owned()]; - pull_partner.ts_pull_token = Some(Redacted::new("outbound-token".to_owned())); - let mut registry = - PartnerRegistry::from_config(&[pull_partner]).expect("should build registry"); - - registry.mark_provider_owned("pull.example.com", None); - - assert!( - registry.pull_enabled_partners().is_empty(), - "should not pull sources an identity provider owns" - ); - } - #[test] fn pull_enabled_partners_filters_correctly() { let mut pull_partner = make_partner("pull.example.com", &valid_api_token("token-p")); diff --git a/crates/trusted-server-core/src/migration_guards.rs b/crates/trusted-server-core/src/migration_guards.rs index 17b3d624f6..5218b14388 100644 --- a/crates/trusted-server-core/src/migration_guards.rs +++ b/crates/trusted-server-core/src/migration_guards.rs @@ -87,7 +87,6 @@ fn checked_sources() -> &'static [(&'static str, &'static str)] { ("ec/mod.rs", include_str!("ec/mod.rs")), ("ec/partner.rs", include_str!("ec/partner.rs")), ("ec/prebid_eids.rs", include_str!("ec/prebid_eids.rs")), - ("ec/pull_sync.rs", include_str!("ec/pull_sync.rs")), ("ec/rate_limiter.rs", include_str!("ec/rate_limiter.rs")), ("ec/registry.rs", include_str!("ec/registry.rs")), ("edge_cookie.rs", include_str!("edge_cookie.rs")), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index af6484981f..9ffd8aca2c 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -62,7 +62,7 @@ use crate::cache_policy::{ CachePolicy, EdgeCacheHeader, cache_control_headers_are_private_or_no_store, }; use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; -use crate::constants::{COOKIE_SHAREDID, COOKIE_TS_EIDS, HEADER_X_COMPRESS_HINT}; +use crate::constants::{COOKIE_TS_EIDS, HEADER_X_COMPRESS_HINT}; use crate::cookies::handle_request_cookies; use crate::cookies::template_cache_policy::{TemplateCookieDecision, evaluate_cookie_policy}; use crate::creative_opportunities::{ @@ -3452,9 +3452,7 @@ struct EcSnapshotPreloadInput { has_navigation_permit: bool, has_ec_id: bool, has_kv: bool, - marker_valid: bool, auction_needs_row: bool, - eid_cookie_may_need_persistence: bool, privacy_needs_row: bool, snapshot_already_read: bool, } @@ -3468,18 +3466,12 @@ fn should_preload_ec_snapshot(input: &EcSnapshotPreloadInput) -> bool { } let eligible = input.is_navigation && input.is_get && input.has_ec_id && input.has_kv; // Without a permit (no Fetch Metadata, or a prefetch) the navigation - // cannot write, so browser EID persistence no longer needs the row; the - // auction, a withdrawal and pull sync (unless the marker proves it - // complete) still do. + // cannot write and pull sync does not run, so only the auction's stored + // EIDs and a withdrawal still need the row. if !input.has_navigation_permit { - return eligible - && (input.auction_needs_row || input.privacy_needs_row || !input.marker_valid); - } - let marker_can_skip = input.marker_valid - && !input.auction_needs_row - && !input.eid_cookie_may_need_persistence - && !input.privacy_needs_row; - eligible && !marker_can_skip + return eligible && (input.auction_needs_row || input.privacy_needs_row); + } + eligible } /// Rewrites a downstream request into an outbound publisher-origin request. @@ -4984,9 +4976,6 @@ pub async fn handle_publisher_request( .filter(|_| ec_allowed && !ec_context.ec_generated()); let ec_id = ec_id_owned.as_deref(); let cookie_jar = handle_request_cookies(&req)?; - if let Some(registry) = auction.registry { - ec_context.validate_pull_sync_marker(settings, registry); - } let geo = ec_context.geo_info().cloned(); let parsed_origin = url::Url::parse(&settings.publisher.origin_url).change_context( @@ -5311,25 +5300,16 @@ pub async fn handle_publisher_request( let request_method = req.method().clone(); let mut origin_request = Some(req); - let eid_cookie_may_need_persistence = cookie_jar - .as_ref() - .is_some_and(|jar| jar.get(COOKIE_TS_EIDS).is_some() || jar.get(COOKIE_SHAREDID).is_some()); - // This decision also gates the concurrent origin send below. A marker skip - // cannot currently delay the origin behind an auction because marker - // validation requires a non-empty registry, and every such auction sets - // `auction_needs_row` and retains the preload. let should_preload_ec = should_preload_ec_snapshot(&EcSnapshotPreloadInput { is_navigation, is_get, has_navigation_permit: ec_context.has_navigation_permit(), has_ec_id: active_ec_id.is_some(), has_kv: kv.is_some(), - marker_valid: ec_context.pull_sync_marker().is_valid(), auction_needs_row: should_run_auction && auction .registry .is_some_and(|registry| !registry.is_empty()), - eid_cookie_may_need_persistence, privacy_needs_row: crate::ec::consent::ec_consent_withdrawn(&consent_context), // A freshly generated EC already holds an authoritative `Missing` // snapshot: its row is only created by this navigation's write. @@ -8413,9 +8393,7 @@ mod tests { has_navigation_permit: true, has_ec_id: true, has_kv: true, - marker_valid: false, auction_needs_row: false, - eid_cookie_may_need_persistence: false, privacy_needs_row: false, snapshot_already_read: false, }; @@ -8446,16 +8424,14 @@ mod tests { } #[test] - fn ec_snapshot_preload_without_a_permit_ignores_browser_eid_persistence() { + fn ec_snapshot_preload_without_a_permit_reads_only_for_auction_or_withdrawal() { let no_permit = EcSnapshotPreloadInput { is_navigation: true, is_get: true, has_navigation_permit: false, has_ec_id: true, has_kv: true, - marker_valid: true, auction_needs_row: false, - eid_cookie_may_need_persistence: true, privacy_needs_row: false, snapshot_already_read: false, }; @@ -8478,43 +8454,8 @@ mod tests { }), "should read the row for a withdrawal" ); - assert!( - should_preload_ec_snapshot(&EcSnapshotPreloadInput { - marker_valid: false, - ..no_permit - }), - "should read the row for pull sync while the marker is not valid" - ); } - #[test] - fn valid_marker_skips_only_when_no_other_consumer_needs_snapshot() { - let marker_only = EcSnapshotPreloadInput { - is_navigation: true, - is_get: true, - has_navigation_permit: true, - has_ec_id: true, - has_kv: true, - marker_valid: true, - auction_needs_row: false, - eid_cookie_may_need_persistence: false, - privacy_needs_row: false, - snapshot_already_read: false, - }; - assert!(!should_preload_ec_snapshot(&marker_only)); - assert!(should_preload_ec_snapshot(&EcSnapshotPreloadInput { - auction_needs_row: true, - ..marker_only - })); - assert!(should_preload_ec_snapshot(&EcSnapshotPreloadInput { - eid_cookie_may_need_persistence: true, - ..marker_only - })); - assert!(should_preload_ec_snapshot(&EcSnapshotPreloadInput { - privacy_needs_row: true, - ..marker_only - })); - } use crate::auction::types::{AdFormat, AdSlot, MediaType}; use crate::consent::ConsentContext; use crate::ec::kv_backend::test_support::InMemoryEcKv; @@ -8886,171 +8827,6 @@ mod tests { } } - #[derive(Clone, Copy)] - enum MarkerProbe { - Valid, - Malformed, - WrongEc, - Expired, - PartnerSetMismatch, - } - - fn marker_partner(source_domain: &str) -> crate::settings::EcPartner { - crate::settings::EcPartner { - name: format!("Marker partner {source_domain}"), - source_domain: source_domain.to_owned(), - openrtb_atype: crate::settings::EcPartner::default_openrtb_atype(), - bidstream_enabled: true, - api_token: Some(crate::redacted::Redacted::new(format!( - "marker-{source_domain}-api-token-32-bytes-minimum" - ))), - batch_rate_limit: crate::settings::EcPartner::default_batch_rate_limit(), - pull_sync_enabled: true, - pull_sync_url: Some(format!("https://sync.{source_domain}/pull")), - pull_sync_allowed_domains: vec![format!("sync.{source_domain}")], - pull_sync_ttl_sec: crate::settings::EcPartner::default_pull_sync_ttl_sec(), - pull_sync_rate_limit: crate::settings::EcPartner::default_pull_sync_rate_limit(), - ts_pull_token: Some(crate::redacted::Redacted::new("pull-token".to_owned())), - } - } - - async fn run_marker_lookup_probe( - probe: MarkerProbe, - has_eid_cookie: bool, - has_matched_auction_slot: bool, - ) -> usize { - let mut settings = if has_matched_auction_slot { - scheduling_settings() - } else { - create_test_settings() - }; - settings.ec.partners = vec![marker_partner("ssp.example.com")]; - let registry = - PartnerRegistry::from_ec(&settings.ec).expect("should build pull partner registry"); - let http = Arc::new(StubHttpClient::new()); - http.set_concurrent_fanout(true); - http.push_response(200, b"ok".to_vec()); - let lookups = Arc::new(AtomicUsize::new(0)); - let graph = KvIdentityGraph::new(OrderRecordingKv { - inner: InMemoryEcKv::new("marker-store"), - http: Arc::clone(&http), - http_calls_at_lookup: Arc::new(AtomicUsize::new(0)), - lookups: Arc::clone(&lookups), - }); - let services = build_services_with_http_client( - Arc::clone(&http) as Arc - ); - let ec_id = format!("{}.CkId01", "b".repeat(64)); - let marker = match probe { - MarkerProbe::Valid => { - crate::ec::pull_sync_marker::create_marker_for_test(&settings, ®istry, &ec_id) - } - MarkerProbe::Malformed => "not-a-marker".to_owned(), - MarkerProbe::WrongEc => crate::ec::pull_sync_marker::create_marker_for_test( - &settings, - ®istry, - &format!("{}.Other1", "c".repeat(64)), - ), - MarkerProbe::Expired => { - crate::ec::pull_sync_marker::create_marker_for_test_with_expiry( - &settings, - ®istry, - &ec_id, - crate::ec::current_timestamp().saturating_sub(1), - ) - } - MarkerProbe::PartnerSetMismatch => { - let other_registry = - PartnerRegistry::from_config(&[marker_partner("other.example.com")]) - .expect("should build alternate registry"); - crate::ec::pull_sync_marker::create_marker_for_test( - &settings, - &other_registry, - &ec_id, - ) - } - }; - let mut ec_context = EcContext::new_for_test_with_ip( - Some(ec_id), - scheduling_consent(), - Some("203.0.113.7".to_owned()), - ); - ec_context.set_navigation_permit(crate::ec::page_load::NavigationPermit::for_test()); - ec_context.set_pull_sync_marker_for_test( - crate::ec::pull_sync_marker::PullSyncMarkerState::from_cookie(Some(marker)), - ); - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); - let slots = has_matched_auction_slot - .then(scheduling_slot) - .into_iter() - .collect::>(); - let mut request = navigation_request(); - if has_eid_cookie { - request - .headers_mut() - .insert(header::COOKIE, HeaderValue::from_static("ts-eids=present")); - } - - let result = handle_publisher_request( - &settings, - &services, - Some(&graph), - &mut ec_context, - AuctionDispatch { - integration_registry: &IntegrationRegistry::empty_for_tests(), - orchestrator: &orchestrator, - slots: &slots, - registry: Some(®istry), - }, - request, - EdgeCacheHeader::SurrogateControl, - ) - .await; - - assert!(result.is_ok(), "should proxy the origin response"); - lookups.load(Ordering::SeqCst) - } - - #[tokio::test] - async fn valid_completeness_marker_skips_pull_only_snapshot_lookup() { - assert_eq!( - run_marker_lookup_probe(MarkerProbe::Valid, false, false).await, - 0 - ); - } - - #[tokio::test] - async fn invalid_markers_fall_back_to_snapshot_lookup() { - for probe in [ - MarkerProbe::Malformed, - MarkerProbe::WrongEc, - MarkerProbe::Expired, - MarkerProbe::PartnerSetMismatch, - ] { - assert_eq!( - run_marker_lookup_probe(probe, false, false).await, - 1, - "invalid marker should retain the normal preload" - ); - } - } - - #[tokio::test] - async fn valid_marker_does_not_skip_eid_cookie_persistence_lookup() { - assert_eq!( - run_marker_lookup_probe(MarkerProbe::Valid, true, false).await, - 1 - ); - } - - #[tokio::test] - async fn valid_marker_does_not_skip_auction_snapshot_lookup() { - assert_eq!( - run_marker_lookup_probe(MarkerProbe::Valid, false, true).await, - 1 - ); - } - #[tokio::test] async fn concurrent_client_starts_origin_before_ec_lookup() { let outcome = run_scheduling_probe(true, true, true, true).await; diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index b528bffcac..72f5a4a7be 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -524,11 +524,8 @@ pub struct EcPartner { /// Allowlist of domains TS may call for this partner's pull sync. #[serde(default, deserialize_with = "vec_from_seq_or_map")] pub pull_sync_allowed_domains: Vec, - /// Legacy pull-sync refresh interval retained for config compatibility. - /// - /// EC identity entries no longer store per-partner sync timestamps, so - /// this value is not used by the current fill-missing-only pull sync - /// behavior. + /// Legacy pull-sync refresh interval; retained for config compatibility; + /// unused. #[serde( default = "EcPartner::default_pull_sync_ttl_sec", deserialize_with = "from_value_or_str" @@ -691,6 +688,31 @@ impl IdentityProviderConfig { true } + /// Builds an enabled page-phase config with no TCF vendor, no rate limit, + /// default secrets and empty provider settings. + #[must_use] + pub fn new( + id: String, + kind: String, + endpoint: String, + allowed_domains: Vec, + sources: Vec, + ) -> Self { + Self { + id, + kind, + enabled: true, + endpoint, + allowed_domains, + sources, + tcf_vendor_id: None, + collect: CollectTiming::default(), + rate_limit_per_hour: None, + secrets: ProviderSecrets::default(), + settings: serde_json::Map::new(), + } + } + /// The configured implementation kind. #[must_use] pub fn kind(&self) -> &str { @@ -739,7 +761,8 @@ pub struct Ec { #[serde(default)] pub ec_store: Option, - /// Maximum number of concurrent pull-sync requests. + /// Maximum number of concurrent pull-sync requests; retained for config + /// compatibility; unused. #[serde(default = "Ec::default_pull_sync_concurrency")] pub pull_sync_concurrency: usize, diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index c815ebaa2c..69256bdad1 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -17,10 +17,12 @@ pub mod criteo; pub mod id5; pub mod liveintent; pub mod liveramp; +pub mod pull_sync; pub mod sharedid; pub mod uid2; pub mod unifiedid; +use std::num::NonZeroU32; use std::sync::Arc; use std::time::Duration; @@ -36,6 +38,8 @@ use trusted_server_core::settings::{ CollectTiming, Ec, IdentityProviderConfig, ProviderSecrets, Settings, }; +use crate::pull_sync::PullSync; + /// Whether `kind` has a registered implementation. /// #[must_use] @@ -50,6 +54,7 @@ pub fn is_registered_kind(kind: &str) -> bool { | sharedid::SharedId::KIND | criteo::Criteo::KIND | unifiedid::UnifiedId::KIND + | PullSync::KIND ) } @@ -76,7 +81,7 @@ fn excludes_where_gdpr_applies(kind: &str, source: &str) -> bool { } /// Builds the partner registry, marking the sources claimed by enabled -/// providers as provider-owned. +/// providers (including pull-sync partners) as provider-owned. /// /// The sources any configured provider, enabled or disabled, excludes where /// GDPR applies are marked too, so the UIDs already stored for them are @@ -117,11 +122,39 @@ pub fn build( sharedid::SharedId::KIND => build_kind::(config, secrets), criteo::Criteo::KIND => build_kind::(config, secrets), unifiedid::UnifiedId::KIND => build_kind::(config, secrets), + PullSync::KIND => build_kind::(config, secrets), kind => Err(Report::new(ProviderConfigError::UnknownKind) .attach(format!("kind `{kind}` is not registered"))), } } +/// One `pull-sync` provider per partner with pull sync enabled, built from the +/// partner's existing fields with no config change. +/// +/// Pull sync runs after the response has been sent, as it did before it +/// became a provider, so a slow partner never delays the page and a new EC +/// whose row does not exist yet is never pulled. Its hourly rate limit is +/// the partner's `pull_sync_rate_limit` (`None` when it is 0). +pub fn pull_sync_providers(ec: &Ec) -> impl Iterator + '_ { + ec.partners + .iter() + .filter(|partner| partner.pull_sync_enabled) + .filter_map(|partner| { + let endpoint = partner.pull_sync_url.clone()?; + let mut config = IdentityProviderConfig::new( + format!("pull-sync:{}", partner.source_domain), + PullSync::KIND.to_owned(), + endpoint, + partner.pull_sync_allowed_domains.clone(), + vec![partner.source_domain.clone()], + ); + config.secrets.token = partner.ts_pull_token.clone(); + config.collect = CollectTiming::AfterResponse; + config.rate_limit_per_hour = NonZeroU32::new(partner.pull_sync_rate_limit); + Some(config) + }) +} + /// Builds the coordinator for enabled providers collected before the /// response (`collect = "page"`, the default). /// @@ -223,14 +256,18 @@ fn build_phase_coordinator( // Source ownership across both phases is checked when settings load // (`config::validate_identity_providers`), so only this phase's providers // are built and checked here. - let registered = settings + let configured = settings .ec .identity_providers .iter() - .filter(|config| config.enabled && config.collect == timing) + .filter(|config| config.enabled) + .cloned(); + let registered = configured + .chain(pull_sync_providers(&settings.ec)) + .filter(|config| config.collect == timing) .map(|config| { - build(config, &config.secrets) - .map(|provider| RegisteredProvider::new(config.clone(), provider)) + build(&config, &config.secrets) + .map(|provider| RegisteredProvider::new(config, provider)) }) .collect::, Report>>()?; if registered.is_empty() { @@ -380,6 +417,39 @@ mod tests { assert!(coordinator.is_none(), "should build no coordinator"); } + #[test] + fn pull_sync_partner_runs_after_the_response() { + let mut settings = create_test_settings(); + settings.ec.partners = vec![ + toml::from_str( + r#" + name = "Pull partner" + source_domain = "partner.example.com" + pull_sync_enabled = true + pull_sync_url = "https://sync.example.com/pull" + pull_sync_allowed_domains = ["sync.example.com"] + ts_pull_token = "example-pull-token-32-bytes-minimum" + "#, + ) + .expect("should parse partner"), + ]; + + let page = build_coordinator(&settings).expect("should build page providers"); + let partners = Arc::new(partner_registry(&settings.ec).expect("should build partners")); + let after_response = build_after_response_coordinator(&settings, partners) + .expect("should build pull-sync providers") + .expect("should build an after-response coordinator"); + + assert!( + page.is_none(), + "should not run pull sync before the response" + ); + assert!( + !after_response.is_empty(), + "should run pull sync after the response" + ); + } + #[test] fn enabled_unknown_kind_fails_the_build() { let mut settings = create_test_settings(); diff --git a/crates/trusted-server-identity-providers/src/pull_sync.rs b/crates/trusted-server-identity-providers/src/pull_sync.rs new file mode 100644 index 0000000000..404dbfffc8 --- /dev/null +++ b/crates/trusted-server-identity-providers/src/pull_sync.rs @@ -0,0 +1,234 @@ +//! The `pull-sync` provider: the existing per-partner pull sync, run as an +//! identity provider on page-load navigations. + +use std::borrow::Cow; +use std::num::NonZeroUsize; + +use error_stack::Report; +use serde::Deserialize; +use trusted_server_core::ec::eid_types::SourceDomain; +use trusted_server_core::ec::provider::{ + EidLevelFields, EidShape, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, + ProviderCommon, ProviderConfigError, ProviderEid, ProviderError, ProviderExpiry, ProviderKind, + ProviderRequest, ProviderResponse, ProviderUid, SourceCapability, SourceInput, +}; +use trusted_server_core::ec::registry::PULL_SYNC_KIND; +use url::Url; + +/// `pull-sync` kind. +pub struct PullSync; + +/// `pull-sync` has no provider-specific settings; everything comes from the +/// partner's existing pull-sync fields. +#[derive(Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PullSyncSettings {} + +/// A built `pull-sync` provider for one partner. +pub struct PullSyncProvider { + endpoint: Url, + source: SourceDomain, + authorization: Option, + capabilities: ProviderCapabilities, +} + +#[derive(Deserialize)] +struct PullSyncBody<'a> { + #[serde(borrow)] + uid: Option>, +} + +impl ProviderKind for PullSync { + const KIND: &'static str = PULL_SYNC_KIND; + type Settings = PullSyncSettings; + type Provider = PullSyncProvider; + + fn build( + common: &ProviderCommon<'_>, + _settings: PullSyncSettings, + ) -> Result> { + let [source] = common.sources else { + return Err(Report::new(ProviderConfigError::Invalid) + .attach("pull-sync claims exactly one source")); + }; + let authorization = common + .secrets + .token + .as_ref() + .map(|token| { + let mut value = http::HeaderValue::try_from(format!("Bearer {}", token.expose())) + .map_err(|_| Report::new(ProviderConfigError::Invalid))?; + value.set_sensitive(true); + Ok::<_, Report>(value) + }) + .transpose()?; + let one = NonZeroUsize::MIN; + Ok(PullSyncProvider { + endpoint: common.endpoint.clone(), + source: source.clone(), + authorization, + capabilities: ProviderCapabilities { + sources: Box::new([SourceCapability { + source: source.clone(), + input: SourceInput::Anonymous, + }]), + shape: EidShape { + max_eids: one, + max_uids_per_eid: one, + eid_level_fields: EidLevelFields::Ignored, + }, + }, + }) + } +} + +impl IdentityProvider for PullSyncProvider { + fn capabilities(&self) -> &ProviderCapabilities { + &self.capabilities + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + let mut url = self.endpoint.clone(); + url.query_pairs_mut().append_pair("ec_id", lookup.ec_id); + let mut headers = http::HeaderMap::new(); + if let Some(value) = &self.authorization { + headers.insert(http::header::AUTHORIZATION, value.clone()); + } + Ok(ProviderRequest { + method: http::Method::GET, + url, + headers, + body: bytes::Bytes::new(), + context: None, + }) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + let body: PullSyncBody<'body> = serde_json::from_slice(response.body) + .map_err(|_| Report::new(ProviderError::InvalidResponse))?; + let Some(uid) = body.uid.filter(|uid| !uid.trim().is_empty()) else { + return Ok(LookupOutcome::NoMatch); + }; + Ok(LookupOutcome::Found(vec![ProviderEid { + source: Cow::Owned(self.source.as_str().to_owned()), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids: vec![ProviderUid { + id: uid, + atype: None, + ext: None, + expiry: ProviderExpiry::Unspecified, + refresh: None, + }], + }])) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use trusted_server_core::consent::ConsentContext; + use trusted_server_core::ec::provider::{ClientSignals, OwnedEntries}; + use trusted_server_core::redacted::Redacted; + use trusted_server_core::settings::ProviderSecrets; + + fn provider() -> PullSyncProvider { + let endpoint = Url::parse("https://sync.example.com/pull?x=1").expect("should parse"); + let sources = [SourceDomain::try_from("partner.example.com").expect("should parse")]; + let secrets = ProviderSecrets { + token: Some(Redacted::new("example-token".to_owned())), + }; + PullSync::build( + &ProviderCommon { + endpoint: &endpoint, + sources: &sources, + secrets: &secrets, + }, + PullSyncSettings {}, + ) + .expect("should build") + } + + #[test] + fn request_carries_ec_id_and_sensitive_bearer_token() { + let provider = provider(); + let consent = ConsentContext::default(); + let due = [SourceDomain::try_from("partner.example.com").expect("should parse")]; + + let request = provider + .build_request(&LookupInput { + ec_id: "abc.DEF123", + consent: &consent, + client: ClientSignals { + ip: None, + user_agent: None, + }, + publisher_domain: "www.example.com", + due_sources: &due, + current: OwnedEntries::empty(), + hashed_email: None, + }) + .expect("should build request"); + + assert_eq!( + request.url.as_str(), + "https://sync.example.com/pull?x=1&ec_id=abc.DEF123", + "should append ec_id" + ); + let auth = request + .headers + .get(http::header::AUTHORIZATION) + .expect("should send token"); + assert_eq!(auth, "Bearer example-token", "should use bearer auth"); + assert!(auth.is_sensitive(), "should mark the token sensitive"); + } + + #[test] + fn parses_uid_and_treats_null_as_no_match() { + let provider = provider(); + let headers = http::HeaderMap::new(); + let respond = |body: &'static [u8]| ProviderResponse { + status: http::StatusCode::OK, + headers: &headers, + body, + context: None, + }; + + let found = provider + .parse_response(respond(b"{\"uid\":\"u-1\"}")) + .expect("should parse"); + let none = provider + .parse_response(respond(b"{\"uid\":null}")) + .expect("should parse"); + let empty = provider + .parse_response(respond(b"{\"uid\":\" \"}")) + .expect("should parse"); + let invalid = provider.parse_response(respond(b"not json")); + + let LookupOutcome::Found(eids) = found else { + panic!("should find a UID") + }; + assert_eq!( + eids[0].source, "partner.example.com", + "should use the partner source" + ); + assert_eq!(eids[0].uids[0].id, "u-1", "should borrow the UID"); + assert!( + matches!(none, LookupOutcome::NoMatch), + "should treat null as no match" + ); + assert!( + matches!(empty, LookupOutcome::NoMatch), + "should treat an empty UID as no match" + ); + assert!(invalid.is_err(), "should reject invalid JSON"); + } +} From b7e33fc15feef8adb8b3404e7353c239bfb9bd82 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 04:55:54 +0530 Subject: [PATCH 02/20] Mirror provider-owned bidstream UIDs to a JS-readable cookie Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/constants.rs | 4 +- crates/trusted-server-core/src/ec/cookies.rs | 2 +- crates/trusted-server-core/src/ec/finalize.rs | 207 ++++++++++++- crates/trusted-server-core/src/ec/mirror.rs | 280 ++++++++++++++++++ crates/trusted-server-core/src/ec/mod.rs | 28 +- crates/trusted-server-core/src/ec/registry.rs | 34 +++ docs/guide/edge-cookies.md | 22 +- 7 files changed, 558 insertions(+), 19 deletions(-) create mode 100644 crates/trusted-server-core/src/ec/mirror.rs diff --git a/crates/trusted-server-core/src/constants.rs b/crates/trusted-server-core/src/constants.rs index ae6d953004..2cdd5407f8 100644 --- a/crates/trusted-server-core/src/constants.rs +++ b/crates/trusted-server-core/src/constants.rs @@ -1,11 +1,13 @@ use http::header::HeaderName; pub const COOKIE_TS_EC: &str = "ts-ec"; -/// Short-lived signed proof that the current EC row has every pull-partner UID. +/// Retired pull-sync completeness marker; leftover cookies are expired. pub const COOKIE_TS_EC_PULL_COMPLETE: &str = "ts-ec-pull-complete"; /// Cookie written by the Trusted Server JS SDK containing a standard-base64-encoded /// JSON array of Extended User IDs (`[{ source, uids }]`) from identity providers. pub const COOKIE_TS_EIDS: &str = "ts-eids"; +/// JS-readable mirror of provider-owned bidstream UIDs (spec §10). +pub const COOKIE_TS_EIDS_MIRROR: &str = "__Host-ts-eids-mirror"; pub const COOKIE_TS_TESTER: &str = "ts-tester"; pub const COOKIE_SHAREDID: &str = "sharedId"; diff --git a/crates/trusted-server-core/src/ec/cookies.rs b/crates/trusted-server-core/src/ec/cookies.rs index 71e5acb109..d45f1073d5 100644 --- a/crates/trusted-server-core/src/ec/cookies.rs +++ b/crates/trusted-server-core/src/ec/cookies.rs @@ -22,7 +22,7 @@ use crate::constants::COOKIE_TS_EC; use crate::settings::Settings; /// Maximum age for the EC cookie (1 year in seconds). -const COOKIE_MAX_AGE: i32 = 365 * 24 * 60 * 60; +pub(crate) const COOKIE_MAX_AGE: i32 = 365 * 24 * 60 * 60; fn is_allowed_ec_id_char(c: char) -> bool { c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_') diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 23043b3590..cbc313a808 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -13,15 +13,16 @@ use http::Response; use super::consent::{ec_consent_granted, ec_consent_withdrawn}; use crate::consent::{ConsentContext, allows_eid_persistence}; -use crate::constants::COOKIE_TS_EC_PULL_COMPLETE; +use crate::constants::{COOKIE_TS_EC_PULL_COMPLETE, COOKIE_TS_EIDS_MIRROR}; use crate::settings::Settings; use super::EcContext; -use super::cookies::{append_cookie, expire_ec_cookie, set_ec_cookie}; +use super::cookies::{COOKIE_MAX_AGE, append_cookie, expire_ec_cookie, set_ec_cookie}; use super::eid_types::UnixSeconds; use super::generation::{generate_ec_id, is_valid_ec_id}; use super::kv::{KvIdentityGraph, PartnerIdUpdate}; use super::kv_types::KvEntry; +use super::mirror::{MirrorCookie, build_mirror, mirror_consent}; use super::page_load::{PageLoadWrite, RowChange}; use super::prebid_eids::collect_eid_updates; use super::registry::PartnerRegistry; @@ -62,7 +63,7 @@ pub fn ec_finalize_response( sharedid_cookie: Option<&str>, response: &mut Response, ) { - finalize_response_inner( + let navigation = finalize_response_inner( settings, ec_context, kv, @@ -71,9 +72,56 @@ pub fn ec_finalize_response( sharedid_cookie, response, ); + apply_mirror(ec_context, registry, navigation, response); expire_retired_pull_marker(ec_context, response); } +/// Sets or expires the mirror cookie (spec §10). +/// +/// Navigations rebuild it from the row as it stands after the page-load +/// write: the committed row is stored as the `Present` snapshot, so +/// [`super::EcKvSnapshot::entry_for`] returns it, or the read snapshot when no +/// write landed. Without EC and EID consent it is expired on navigations and on any +/// response whose request carried it. +fn apply_mirror( + ec_context: &EcContext, + registry: &PartnerRegistry, + navigation: bool, + response: &mut Response, +) { + let decision = if !mirror_consent(ec_context.consent()) { + MirrorCookie::Expire + } else if navigation { + let row = ec_context + .ec_value() + .and_then(|ec_id| ec_context.kv_snapshot().entry_for(ec_id)); + build_mirror( + row, + ec_context.consent(), + registry, + UnixSeconds::new(current_timestamp()), + ) + } else { + return; + }; + match decision { + MirrorCookie::Set(value) => append_cookie( + response, + &format!( + "{COOKIE_TS_EIDS_MIRROR}={}; Path=/; Secure; SameSite=Lax; Max-Age={COOKIE_MAX_AGE}", + value.as_str() + ), + ), + MirrorCookie::Expire if ec_context.mirror_cookie_present() || navigation => { + append_cookie( + response, + &format!("{COOKIE_TS_EIDS_MIRROR}=; Path=/; Secure; SameSite=Lax; Max-Age=0"), + ); + } + MirrorCookie::Expire => {} + } +} + /// Expires a leftover pull-sync completeness marker; the marker is retired. fn expire_retired_pull_marker(ec_context: &EcContext, response: &mut Response) { if ec_context.pull_marker_present() { @@ -94,11 +142,12 @@ fn finalize_response_inner( eids_cookie: Option<&str>, sharedid_cookie: Option<&str>, response: &mut Response, -) { +) -> bool { let (permit, after_response) = ec_context .take_navigation_permit() .and_then(|permit| permit.confirm_with_after_response(response.status())) .unzip(); + let navigation = permit.is_some(); let consent_allows_ec = ec_consent_granted(ec_context.consent()); let consent_withdrawn = ec_consent_withdrawn(ec_context.consent()); @@ -111,7 +160,7 @@ fn finalize_response_inner( consent_withdrawn, response, ); - return; + return navigation; } if let (Some(graph), Some(ec_id)) = (kv, ec_context.ec_value().map(str::to_owned)) { @@ -152,6 +201,7 @@ fn finalize_response_inner( // mint a browser cookie with no backing identity-graph row. log::info!("Skipping generated EC response write because KV graph is unavailable"); } + navigation } /// Shared inputs for the navigation's single write. @@ -841,7 +891,7 @@ mod tests { ); assert!( - get_header(&response, "set-cookie").is_none(), + !ec_cookie_emitted(&response), "should keep the EC cookie when the tombstone write fails" ); assert!( @@ -1817,7 +1867,7 @@ mod tests { ); assert!( - get_header(&response, "set-cookie").is_none(), + !ec_cookie_emitted(&response), "must not emit a cookie without an authoritative backing row" ); } @@ -1893,6 +1943,134 @@ mod tests { // Orphan-recovery gating and two-ID withdrawal // ----------------------------------------------------------------------- + fn mirror_cookies(response: &Response) -> Vec { + response + .headers() + .get_all(http::header::SET_COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .filter(|cookie| cookie.starts_with("__Host-ts-eids-mirror=")) + .map(str::to_owned) + .collect() + } + + fn provider_found(source: &str) -> LookupResults { + let uid = KvUid::new( + UidValue::try_from("provider-uid").expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::Never, + ); + // Made for the EC ID `run_mirror_navigation` seeds. + LookupResults::new( + &sample_ec_id("mirr01"), + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Found { + kept: 1, + dropped: DropCounts::default(), + }, + entries: vec![( + SourceDomain::try_from(source).expect("should build source"), + vec![KvEid::new(EidFields::default(), vec![uid])], + )], + cleared: Vec::new(), + }], + ) + } + + fn run_mirror_navigation(lookups: LookupResults) -> Response { + let settings = create_test_settings(); + let ec_id = sample_ec_id("mirr01"); + let graph = KvIdentityGraph::in_memory("mirror_store"); + let live = KvEntry::new( + &granting_consent(), + None, + current_timestamp(), + &settings.publisher.domain, + ); + graph + .create(&ec_id, &live) + .expect("should seed the live row"); + let mut ec_context = returning_user_context(&ec_id, graph.load_snapshot(&ec_id), true); + ec_context.set_lookup_results(lookups); + let mut registry = PartnerRegistry::from_config(&[make_partner("ids.example.com")]) + .expect("should build registry"); + registry.mark_provider_owned("ids.example.com", None); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry, + None, + None, + &mut response, + ); + response + } + + #[test] + fn navigation_sets_mirror_from_committed_row() { + let response = run_mirror_navigation(provider_found("ids.example.com")); + + let cookies = mirror_cookies(&response); + assert_eq!(cookies.len(), 1, "should set one mirror cookie"); + assert!( + !cookies[0].contains("Max-Age=0") && cookies[0].contains("Max-Age="), + "should set the mirror with a non-zero Max-Age" + ); + assert!( + !cookies[0].contains("HttpOnly") && !cookies[0].contains("Domain="), + "should be JS-readable and host-only" + ); + } + + #[test] + fn navigation_without_provider_uids_expires_mirror() { + let response = run_mirror_navigation(LookupResults::default()); + + let cookies = mirror_cookies(&response); + assert_eq!(cookies.len(), 1, "should emit one mirror cookie"); + assert!( + cookies[0].contains("Max-Age=0"), + "should expire the mirror when no provider UID qualifies" + ); + } + + #[test] + fn withdrawn_consent_expires_mirror_on_any_response() { + let settings = create_test_settings(); + let consent = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + source: ConsentSource::Cookie, + ..Default::default() + }; + let mut ec_context = make_context_with_consent(None, None, false, false, consent); + ec_context.set_mirror_cookie_present_for_test(true); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + None, + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + let cookies = mirror_cookies(&response); + assert_eq!(cookies.len(), 1, "should emit one mirror cookie"); + assert!( + cookies[0].contains("Max-Age=0"), + "should expire the mirror without a navigation" + ); + } + fn granting_consent() -> ConsentContext { ConsentContext { jurisdiction: Jurisdiction::NonRegulated, @@ -1921,6 +2099,17 @@ mod tests { ec } + /// Whether the response sets or expires the `ts-ec` cookie; other + /// cookies, such as the mirror, are ignored. + fn ec_cookie_emitted(response: &Response) -> bool { + response + .headers() + .get_all(http::header::SET_COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .any(|cookie| cookie.starts_with("ts-ec=")) + } + fn assert_did_not_rotate(ec_context: &EcContext, orphan: &str, response: &Response) { assert_eq!( ec_context.ec_value(), @@ -1929,7 +2118,7 @@ mod tests { ); assert!(!ec_context.ec_generated(), "must not mark a rotated EC"); assert!( - get_header(response, "set-cookie").is_none(), + !ec_cookie_emitted(response), "must not emit a replacement cookie" ); } @@ -2517,7 +2706,7 @@ mod tests { ); assert!( - get_header(&response, "set-cookie").is_none(), + !ec_cookie_emitted(&response), "should not issue an EC whose row was not created" ); } diff --git a/crates/trusted-server-core/src/ec/mirror.rs b/crates/trusted-server-core/src/ec/mirror.rs new file mode 100644 index 0000000000..72d678e05c --- /dev/null +++ b/crates/trusted-server-core/src/ec/mirror.rs @@ -0,0 +1,280 @@ +//! The JS-readable mirror of provider-owned bidstream UIDs (spec §10). + +use base64::Engine as _; +use serde_json::{Map, Value, json}; + +use crate::consent::ConsentContext; +use crate::consent::allows_eid_persistence; +use crate::ec::consent::ec_consent_granted; +use crate::ec::eid_types::{Expiry, UnixSeconds}; +use crate::ec::kv_types::{KvEid, KvEntry, KvUid}; +use crate::ec::registry::PartnerRegistry; + +/// Largest encoded mirror value. +pub const MAX_MIRROR_BYTES: usize = 3584; + +/// What to do with the mirror cookie on this response. +#[derive(Debug, PartialEq, Eq)] +pub enum MirrorCookie { + /// Set the cookie to this value. + Set(MirrorValue), + /// Expire the cookie. + Expire, +} + +/// An encoded mirror value within [`MAX_MIRROR_BYTES`]. +#[derive(Debug, PartialEq, Eq)] +pub struct MirrorValue(String); + +impl MirrorValue { + /// The encoded cookie value. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +/// Builds the mirror from the row as it stands after this navigation. +/// +/// Holds live, provider-written UIDs of `bidstream_enabled` partners, in +/// partner config order, as unpadded base64url JSON +/// `{"v":1,"eids":[...]}`. UIDs that would push the value past +/// [`MAX_MIRROR_BYTES`] are skipped. Returns [`MirrorCookie::Expire`] when +/// consent does not allow the EC or EID persistence, the row is missing or withdrawn, +/// or nothing qualifies. +/// +/// # Performance +/// +/// Re-encodes once per kept UID; the size cap bounds this to a few dozen +/// small encodes per navigation. +#[must_use] +pub fn build_mirror( + row: Option<&KvEntry>, + consent: &ConsentContext, + partners: &PartnerRegistry, + now: UnixSeconds, +) -> MirrorCookie { + let Some(row) = row.filter(|row| row.consent.ok && mirror_consent(consent)) else { + return MirrorCookie::Expire; + }; + let mut eids: Vec = Vec::new(); + let mut encoded: Option = None; + for partner in partners + .in_config_order() + .filter(|partner| partner.bidstream_enabled) + { + let Some(record) = row.ids.get(&partner.source_domain) else { + continue; + }; + for eid in record.eids() { + let entry_index = eids.len(); + eids.push(eid_json(&partner.source_domain, eid)); + let mut kept = 0_usize; + for uid in eid + .uids() + .iter() + .filter(|uid| uid.owner().is_some() && uid.is_live(now)) + { + uids_mut(&mut eids[entry_index]).push(uid_json(uid)); + if let Some(value) = encode(&eids) { + encoded = Some(value); + kept += 1; + } else { + uids_mut(&mut eids[entry_index]).pop(); + } + } + if kept == 0 { + eids.pop(); + } + } + } + match encoded { + Some(value) if !eids.is_empty() => MirrorCookie::Set(MirrorValue(value)), + _ => MirrorCookie::Expire, + } +} + +/// Whether consent allows the mirror: EC consent and EID persistence. +#[must_use] +pub(crate) fn mirror_consent(consent: &ConsentContext) -> bool { + ec_consent_granted(consent) && allows_eid_persistence(consent) +} + +fn uids_mut(eid: &mut Value) -> &mut Vec { + eid["uids"] + .as_array_mut() + .expect("should hold a uids array") +} + +fn encode(eids: &[Value]) -> Option { + let json = serde_json::to_vec(&json!({ "v": 1, "eids": eids })).ok()?; + let value = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(json); + (value.len() <= MAX_MIRROR_BYTES).then_some(value) +} + +fn eid_json(source: &str, eid: &KvEid) -> Value { + let fields = eid.fields(); + let mut object = Map::new(); + object.insert("source".to_owned(), Value::from(source)); + if let Some(inserter) = &fields.inserter { + object.insert("inserter".to_owned(), Value::from(inserter.as_ref())); + } + if let Some(matcher) = &fields.matcher { + object.insert("matcher".to_owned(), Value::from(matcher.as_ref())); + } + if let Some(mm) = fields.mm { + object.insert("mm".to_owned(), Value::from(i32::from(mm))); + } + if let Some(ext) = &fields.ext { + object.insert("ext".to_owned(), ext.to_value()); + } + object.insert("uids".to_owned(), Value::Array(Vec::new())); + Value::Object(object) +} + +fn uid_json(uid: &KvUid) -> Value { + let mut object = Map::new(); + object.insert("id".to_owned(), Value::from(uid.id().as_str())); + object.insert("atype".to_owned(), Value::from(uid.atype().get())); + if let Some(ext) = uid.ext() { + object.insert("ext".to_owned(), ext.to_value()); + } + if let Expiry::At(at) = uid.expiry() { + object.insert("exp".to_owned(), Value::from(at.get())); + } + Value::Object(object) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ec::eid_types::{AgentType, EidFields, ProviderId, UidValue}; + use crate::ec::kv_types::KvPartnerId; + + fn partners(bidstream: bool) -> PartnerRegistry { + let partner = serde_json::from_value(json!({ + "name": "ids", "source_domain": "ids.example.com", "bidstream_enabled": bidstream + })) + .expect("should build partner"); + PartnerRegistry::from_config(&[partner]).expect("should build registry") + } + + fn uid(id: &str, expiry: Expiry) -> KvUid { + KvUid::new( + UidValue::try_from(id).expect("should build"), + AgentType::try_from(1).expect("should build"), + None, + Some(ProviderId::new("p")), + expiry, + ) + } + + fn row(uids: Vec) -> KvEntry { + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + let record = KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), uids)], + UnixSeconds::new(0), + ) + .expect("should build record"); + entry.ids.insert("ids.example.com".to_owned(), record); + entry + } + + fn granting() -> ConsentContext { + ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + } + } + + fn decode(value: &MirrorValue) -> Value { + let bytes = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(value.as_str()) + .expect("should decode"); + serde_json::from_slice(&bytes).expect("should parse") + } + + #[test] + fn mirrors_live_provider_uids_with_exp() { + let entry = row(vec![ + uid("live", Expiry::At(UnixSeconds::new(2000))), + uid("old", Expiry::At(UnixSeconds::new(10))), + ]); + + let MirrorCookie::Set(value) = build_mirror( + Some(&entry), + &granting(), + &partners(true), + UnixSeconds::new(1000), + ) else { + panic!("should set the mirror"); + }; + + assert_eq!( + decode(&value), + json!({ "v": 1, "eids": [{ "source": "ids.example.com", "uids": [{ "id": "live", "atype": 1, "exp": 2000 }] }] }), + "should hold only the live UID with its exp" + ); + } + + #[test] + fn expires_without_consent_bidstream_or_uids() { + let entry = row(vec![uid("live", Expiry::Never)]); + let now = UnixSeconds::new(1000); + + assert!( + matches!( + build_mirror( + Some(&entry), + &ConsentContext::default(), + &partners(true), + now + ), + MirrorCookie::Expire + ), + "should expire without consent" + ); + assert!( + matches!( + build_mirror(Some(&entry), &granting(), &partners(false), now), + MirrorCookie::Expire + ), + "should expire for non-bidstream sources" + ); + assert!( + matches!( + build_mirror(None, &granting(), &partners(true), now), + MirrorCookie::Expire + ), + "should expire without a row" + ); + } + + #[test] + fn skips_uids_that_do_not_fit() { + let big: Vec = (0..20) + .map(|i| uid(&format!("{i:02}{}", "x".repeat(300)), Expiry::Never)) + .collect(); + let entry = row(big); + + let MirrorCookie::Set(value) = build_mirror( + Some(&entry), + &granting(), + &partners(true), + UnixSeconds::new(1000), + ) else { + panic!("should set the mirror"); + }; + + assert!( + value.as_str().len() <= MAX_MIRROR_BYTES, + "should fit the size limit" + ); + let kept = decode(&value)["eids"][0]["uids"] + .as_array() + .expect("should hold uids") + .len(); + assert!(kept > 0 && kept < 20, "should keep a prefix that fits"); + } +} diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index e1f750f6d1..fa7fe3f85b 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -33,6 +33,7 @@ //! - [`write_plan`] — Plans a navigation's single row change //! - [`provider`] — Identity provider interface and configuration //! - [`coordinator`] — Server-side identity lookup coordination +//! - [`mirror`] — JS-readable mirror of provider-owned bidstream UIDs mod auth; @@ -51,6 +52,7 @@ pub mod identify; pub mod kv; pub mod kv_backend; pub mod kv_types; +pub mod mirror; pub mod page_load; pub mod partner; pub mod prebid_eids; @@ -82,7 +84,7 @@ use error_stack::Report; use http::Request; use crate::consent::{self as consent_mod, ConsentContext, ConsentPipelineInput}; -use crate::constants::{COOKIE_TS_EC, COOKIE_TS_EC_PULL_COMPLETE}; +use crate::constants::{COOKIE_TS_EC, COOKIE_TS_EC_PULL_COMPLETE, COOKIE_TS_EIDS_MIRROR}; use crate::cookies::handle_request_cookies; use crate::ec::cookies::ec_id_has_only_allowed_chars; use crate::error::TrustedServerError; @@ -191,6 +193,8 @@ struct RequestEc { cookie_ec: Option, /// Whether the retired pull-sync completeness marker is present. pull_marker_present: bool, + /// Whether the mirror cookie is present. + mirror_cookie_present: bool, /// The parsed cookie jar (retained for consent pipeline input). jar: Option, } @@ -210,10 +214,14 @@ fn parse_ec_from_request(req: &Request) -> Result>, /// Whether the request carried the retired pull-sync marker cookie. pull_marker_present: bool, + /// Whether the request carried the mirror cookie. + mirror_cookie_present: bool, /// Allowed returning-user EID persistence source, assigned only after request filters pass. eid_sync_source: Option, /// Navigation permit issued by the router for page-load navigations. @@ -399,6 +409,7 @@ impl EcContext { recovery_eligible: false, client_eids: None, pull_marker_present: parsed.pull_marker_present, + mirror_cookie_present: parsed.mirror_cookie_present, eid_sync_source: None, navigation_permit: PermitSlot::default(), after_response_permit: AfterResponsePermitSlot::default(), @@ -594,6 +605,18 @@ impl EcContext { self.pull_marker_present } + /// Whether the request carried the mirror cookie. + #[must_use] + pub(crate) fn mirror_cookie_present(&self) -> bool { + self.mirror_cookie_present + } + + /// Marks the mirror cookie as present, for tests. + #[cfg(test)] + pub(crate) fn set_mirror_cookie_present_for_test(&mut self, present: bool) { + self.mirror_cookie_present = present; + } + /// Marks the retired pull-sync marker cookie as present, for tests. #[cfg(test)] pub(crate) fn set_pull_marker_present_for_test(&mut self, present: bool) { @@ -703,6 +726,7 @@ impl EcContext { recovery_eligible: false, client_eids: None, pull_marker_present: false, + mirror_cookie_present: false, eid_sync_source: None, navigation_permit: PermitSlot::default(), after_response_permit: AfterResponsePermitSlot::default(), @@ -732,6 +756,7 @@ impl EcContext { recovery_eligible: false, client_eids: None, pull_marker_present: false, + mirror_cookie_present: false, eid_sync_source: None, navigation_permit: PermitSlot::default(), after_response_permit: AfterResponsePermitSlot::default(), @@ -764,6 +789,7 @@ impl EcContext { recovery_eligible: false, client_eids: None, pull_marker_present: false, + mirror_cookie_present: false, eid_sync_source: None, navigation_permit: PermitSlot::default(), after_response_permit: AfterResponsePermitSlot::default(), diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index 4174c03748..928afcb0f9 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -93,6 +93,8 @@ impl PartnerConfig { pub struct PartnerRegistry { by_source_domain: HashMap, by_api_key_hash: HashMap, + /// Normalized source domains in `[[ec.partners]]` order. + order: Vec, } impl PartnerRegistry { @@ -178,6 +180,7 @@ impl PartnerRegistry { pub fn from_config(partners: &[EcPartner]) -> Result> { let mut by_source_domain = HashMap::with_capacity(partners.len()); let mut by_api_key_hash = HashMap::with_capacity(partners.len()); + let mut order = Vec::with_capacity(partners.len()); for partner in partners { let normalized_source = normalize_partner_source_domain(&partner.source_domain) @@ -231,12 +234,14 @@ impl PartnerRegistry { if let Some(api_key_hash) = api_key_hash { by_api_key_hash.insert(api_key_hash, normalized_source.clone()); } + order.push(normalized_source.clone()); by_source_domain.insert(normalized_source, config); } Ok(Self { by_source_domain, by_api_key_hash, + order, }) } @@ -305,6 +310,7 @@ impl PartnerRegistry { Self { by_source_domain: HashMap::new(), by_api_key_hash: HashMap::new(), + order: Vec::new(), } } @@ -391,6 +397,13 @@ impl PartnerRegistry { .collect() } + /// Partners in `[[ec.partners]]` order. + pub fn in_config_order(&self) -> impl Iterator { + self.order + .iter() + .filter_map(|source| self.by_source_domain.get(source)) + } + /// Returns an iterator over all configured partners. /// /// Iteration order is unspecified; callers that need determinism should @@ -724,6 +737,27 @@ mod tests { ); } + #[test] + fn in_config_order_preserves_partner_order() { + let partners = vec![ + make_partner("c.example.com", &valid_api_token("token-c")), + make_partner("a.example.com", &valid_api_token("token-a")), + make_partner("b.example.com", &valid_api_token("token-b")), + ]; + let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); + + let order: Vec<&str> = registry + .in_config_order() + .map(|partner| partner.source_domain.as_str()) + .collect(); + + assert_eq!( + order, + ["c.example.com", "a.example.com", "b.example.com"], + "should keep config order" + ); + } + #[test] fn empty_config_builds_empty_registry() { let registry = PartnerRegistry::from_config(&[]).expect("should build empty registry"); diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index c137d6ddbe..14781bd465 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -65,7 +65,6 @@ sequenceDiagram TS-->>B: Response
(no cookie refresh) end - Note over TS,KV: Phase 4: Post-send (background)
Dispatch pull-sync to partners ``` ### Response Finalization @@ -202,7 +201,7 @@ sequenceDiagram participant DSP as Downstream Partners / DSPs alt Pull sync seeds partner UID - TS->>DSP: Background pull sync request
(EC ID + consent context) + TS->>DSP: Pull sync lookup on a page navigation
(EC ID + consent context) DSP-->>TS: Partner UID for EC TS->>KV: Upsert ids[source_domain] = UID else Prebid sync seeds browser EIDs @@ -264,15 +263,24 @@ The relevant OpenRTB structure forwarded to Prebid Server and downstream partner Server-resolved EIDs and current-request Prebid EIDs are deduplicated by `source + uid.id`. Browser-supplied EIDs from the `ts-eids` or `sharedId` cookie only fill partner UIDs that are missing from KV (the `/auction` request body goes to bidders only and is never stored); a different stored UID is kept, as described in [Prebid EID Flow](#prebid-eid-flow). -### Pull-Sync Completeness Marker +### Pull Sync on Navigations -When the identity graph contains a UID for every pull-enabled partner, Trusted Server sets a signed, host-only `ts-ec-pull-complete` cookie. The cookie contains no partner UID or EC ID. It authenticates a one-hour expiration and a fingerprint of the current pull-partner source-domain set, bound to the active EC ID with key material derived from `ec.passphrase`. +Pull sync runs as an identity provider during top-level page navigations, within the shared identity lookup budget, and its result lands in the navigation's single KV write. A pull-sync partner's source is provider-owned: browser EID ingestion skips it and batch sync rejects its mappings with `provider_owned`. -A valid marker avoids a KV lookup only when pull-sync completeness is the sole reason to inspect the row. Auctions that need stored EIDs, browser EID-cookie ingestion, explicit withdrawal, and generation continue to use KV. The marker acts as recent proof that the row existed, so deletion of a previously complete row is not detected until the marker expires, at most one hour after issuance. Partner-set changes, passphrase rotation, malformed values, and expiration invalidate the marker and restore the normal lookup and orphan-recovery path. +The former `ts-ec-pull-complete` completeness marker is retired. A response to a request that still carries it expires the leftover cookie. -Pull sync runs after response delivery, so a partner response that fills the last missing UID cannot set the marker on that already-sent response. A later eligible request verifies the completed row and issues the marker. Explicit withdrawal always expires any present `ts-ec-pull-complete` marker, and expires `ts-ec` once the row is tombstoned or proven absent. While the tombstone cannot be written (KV unavailable, CAS retries exhausted, or a row that cannot be read), `ts-ec` is kept so the next request that carries it retries the withdrawal. +Explicit withdrawal expires `ts-ec` once the row is tombstoned or proven absent. While the tombstone cannot be written (KV unavailable, CAS retries exhausted, or a row that cannot be read), `ts-ec` is kept so the next request that carries it retries the withdrawal. -Issuing or expiring the marker adds `Set-Cookie` to the outgoing response. Cache-privacy handling makes an otherwise shareable response private when that happens. A valid marker is not refreshed on each request, so this cost is limited to responses that establish or clear marker state in exchange for avoiding later KV reads. +### EID Mirror Cookie + +Trusted Server mirrors provider-written UIDs for `bidstream_enabled` partners into a JavaScript-readable cookie so client-side code can use them: + +- **Name**: `__Host-ts-eids-mirror` +- **Attributes**: `Path=/; Secure; SameSite=Lax; Max-Age=`, host-only (no `Domain`) and not `HttpOnly` +- **Contents**: unpadded base64url JSON `{"v":1,"eids":[{"source":"ids.example.com","uids":[{"id":"...","atype":1,"exp":1767225600}]}]}`, partners in `[[ec.partners]]` order, at most 3584 bytes; UIDs that do not fit are left out +- **Updates**: every page navigation rebuilds it from the row after that navigation's write, and expires it when no live provider UID qualifies. Without EC and EID consent it is expired on navigations and on any response whose request carried it. + +Readers must drop UIDs whose `exp` (Unix seconds) has passed and strip `exp` before using a UID. ## Configuration From 3679de3200816b0c5521f185c964b418e4486f7d Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 13:07:08 +0530 Subject: [PATCH 03/20] Make mirroring best effort and fix pull sync validation Signed-off-by: dhruv8sh --- .../trusted-server-adapter-fastly/src/app.rs | 16 +- crates/trusted-server-core/src/config.rs | 43 ++- .../src/ec/coordinator/mod.rs | 192 ++++++++++++-- .../src/ec/coordinator/results.rs | 4 + crates/trusted-server-core/src/ec/finalize.rs | 107 +++++++- crates/trusted-server-core/src/ec/mirror.rs | 244 +++++++++++++++--- crates/trusted-server-core/src/ec/provider.rs | 3 +- crates/trusted-server-core/src/settings.rs | 42 ++- .../src/lib.rs | 6 +- docs/guide/configuration.md | 2 +- docs/guide/edge-cookies.md | 3 +- 11 files changed, 571 insertions(+), 91 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 386f6e7466..cadd3d4665 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -373,8 +373,6 @@ pub(crate) struct EcFinalizeState { pub(crate) use_finalize_kv: bool, pub(crate) eids_cookie: Option, pub(crate) sharedid_cookie: Option, - #[cfg_attr(not(test), allow(dead_code))] - pub(crate) is_real_browser: bool, /// Per-request services carried to the entry point so the pull-sync /// dispatcher can reuse the same platform HTTP client. pub(crate) services: RuntimeServices, @@ -562,7 +560,6 @@ impl EcRequestState { use_finalize_kv: self.finalize_kv_graph.is_some(), eids_cookie: self.eids_cookie, sharedid_cookie: self.sharedid_cookie, - is_real_browser: self.is_real_browser, services: self.services, } } @@ -937,8 +934,6 @@ async fn run_named_route( /// graph + partner registry + rate limiter, with a default EC context for /// response finalization. fn run_batch_sync(state: &AppState, services: &RuntimeServices, req: Request) -> Response { - let device_signals = device_signals_for(&req); - let is_real_browser = device_signals.looks_like_browser(); let eids_cookie = crate::extract_cookie_value(&req, COOKIE_TS_EIDS); let sharedid_cookie = crate::extract_cookie_value(&req, COOKIE_SHAREDID); @@ -956,7 +951,6 @@ fn run_batch_sync(state: &AppState, services: &RuntimeServices, req: Request) -> use_finalize_kv: false, eids_cookie, sharedid_cookie, - is_real_browser, services: services.clone(), }); response @@ -3476,7 +3470,10 @@ mod tests { .get::() .expect("fallback response should carry EcFinalizeState"); assert!( - finalize.is_real_browser, + finalize + .ec_context + .device_signals() + .is_some_and(DeviceSignals::looks_like_browser), "browser-shaped JA4/H2 signals from the request extension must mark the request as a real browser" ); } @@ -3495,7 +3492,10 @@ mod tests { .get::() .expect("fallback response should carry EcFinalizeState"); assert!( - !finalize.is_real_browser, + !finalize + .ec_context + .device_signals() + .is_some_and(DeviceSignals::looks_like_browser), "a request without captured device signals should not be classified as a real browser" ); } diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index f729a22f2e..caf324e02a 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -459,7 +459,24 @@ fn validate_identity_providers(ec: &Ec) -> Result<(), Report report.change_context(TrustedServerError::Configuration { message: format!("ec.identity_providers: {detail}"), }) - }) + })?; + // Pull-sync partners run as providers too; a source claimed by both would + // fail the coordinator build on every request. + for config in ec.identity_providers.iter().filter(|config| config.enabled) { + if let Some(source) = config.sources.iter().find(|source| { + partners + .find_by_source_domain(source) + .is_some_and(|partner| partner.pull_sync_enabled && partner.pull_sync_url.is_some()) + }) { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "ec.identity_providers: source `{source}` of provider `{}` is also claimed by a pull-sync partner", + config.id + ), + })); + } + } + Ok(()) } fn validate_js_asset_proxy_config(settings: &Settings) -> Result<(), Report> { @@ -1667,6 +1684,30 @@ gam_network_id = "99999" ); } + #[test] + fn runtime_validation_rejects_source_claimed_by_provider_and_pull_sync() { + let mut settings = settings_with_identity_provider(serde_json::json!({})); + settings.ec.partners = vec![ + serde_json::from_value(serde_json::json!({ + "name": "ids", + "source_domain": "ids.example.com", + "pull_sync_enabled": true, + "pull_sync_url": "https://sync.example.com/pull", + "pull_sync_allowed_domains": ["sync.example.com"], + "ts_pull_token": "example-pull-token-32-bytes-minimum", + })) + .expect("should build partner"), + ]; + + let err = validate_settings_for_runtime(&settings) + .expect_err("should reject a source claimed twice"); + + assert!( + err.to_string().contains("pull-sync partner"), + "should explain the double claim: {err:?}" + ); + } + #[test] fn runtime_validation_rejects_provider_source_without_partner() { let settings = settings_with_identity_provider(serde_json::json!({ diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 99821f5692..9b875f20b6 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -9,7 +9,7 @@ pub mod transport; mod validate; use std::collections::HashSet; -use std::num::{NonZeroU16, NonZeroU32}; +use std::num::{NonZeroU16, NonZeroU32, NonZeroUsize}; use std::sync::Arc; use std::time::Duration; @@ -34,7 +34,7 @@ use crate::ec::provider::{ destination_allowed, parse_sources, validate_enabled_configs, }; use crate::ec::rate_limiter::RateLimiter; -use crate::ec::registry::PartnerRegistry; +use crate::ec::registry::{PULL_SYNC_KIND, PartnerRegistry}; use crate::settings::IdentityProviderConfig; @@ -178,6 +178,9 @@ struct ProviderPolicy { allowed_domains: Box<[Box]>, tcf_vendor_id: Option, rate_limit_per_hour: Option, + /// Whether this is a pull-sync partner, counted against + /// [`ProviderCoordinator::with_pull_sync_concurrency`]. + pull_sync: bool, } /// The production coordinator, built once at startup. @@ -189,6 +192,8 @@ pub struct ProviderCoordinator

> { /// Partner config: `openrtb_atype` for validation. partners: Arc, budget: Duration, + /// Most pull-sync lookups dispatched per navigation; `None` is unlimited. + pull_sync_concurrency: Option, } impl ProviderCoordinator

{ @@ -230,9 +235,19 @@ impl ProviderCoordinator

{ providers, partners, budget, + pull_sync_concurrency: None, }) } + /// Caps the pull-sync lookups dispatched per navigation + /// (`ec.pull_sync_concurrency`). Due pull-sync partners past the cap are + /// skipped as rate limited and retried on a later navigation. + #[must_use] + pub fn with_pull_sync_concurrency(mut self, max: Option) -> Self { + self.pull_sync_concurrency = max; + self + } + fn register( registered: RegisteredProvider

, ) -> Result, Report> { @@ -261,6 +276,7 @@ impl ProviderCoordinator

{ .collect(), tcf_vendor_id: config.tcf_vendor_id, rate_limit_per_hour: config.rate_limit_per_hour, + pull_sync: config.kind() == PULL_SYNC_KIND, }, provider, }) @@ -294,7 +310,8 @@ impl ProviderCoordinator

{ } /// Claimed sources lacking a live provider-written UID (a browser - /// fallback does not count), minus sources needing input V1 + /// fallback does not count; for pull sync, a live owned UID), minus + /// sources needing input V1 /// never has; the flag reports whether any source was removed for input. fn due_sources( registered: &Registered

, @@ -305,16 +322,31 @@ impl ProviderCoordinator

{ let mut input_blocked = false; let mut due = Vec::new(); for source in ®istered.policy.sources { - let record = entry - .and_then(|entry| entry.ids.get(source.as_str())) - .filter(|record| record.is_provider_written()); + let record = entry.and_then(|entry| entry.ids.get(source.as_str())); // A UID whose provider asked for a refresh makes its source due; // the stored refresh state is the lookup's input. - if record.is_some_and(|record| record.refresh_due_for(®istered.id, now)) { + if record + .filter(|record| record.is_provider_written()) + .is_some_and(|record| record.refresh_due_for(®istered.id, now)) + { due.push(source.clone()); continue; } - if record.is_some_and(|record| record.has_live_uid(now)) { + // Legacy pull sync wrote single-UID records with no owner; a + // pull-sync provider treats them as its own and refreshes them so + // they gain an owner and are mirrored. + let held = record.is_some_and(|record| { + if registered.policy.pull_sync { + record + .eids() + .iter() + .flat_map(KvEid::uids) + .any(|uid| uid.owner().is_some() && uid.is_live(now)) + } else { + record.is_provider_written() && record.has_live_uid(now) + } + }); + if held { continue; } let needs_input = capabilities @@ -515,6 +547,9 @@ impl IdentityCoordinator for ProviderCoordinator

{ if self.providers.is_empty() { return pending; } + // A failed read cannot rule out a withdrawal tombstone, so no provider + // is contacted. + let snapshot_failed = matches!(lookup.snapshot, EcKvSnapshot::Failed { .. }); let entry = lookup.snapshot.entry_for(lookup.ec_id); // Missing EC or EID consent, a withdrawal tombstone, an unreadable // row (which may hide a tombstone) or no row at all skips every @@ -526,12 +561,17 @@ impl IdentityCoordinator for ProviderCoordinator

{ && entry.is_some_and(|entry| entry.consent.ok); let deadline = Deadline::after(self.budget); pending.deadline = Some(deadline); + let mut pull_sync_dispatched = 0_usize; for (index, registered) in self.providers.iter().enumerate() { let provider = ProviderIndex::new(index); let skip = |reason| Tagged { provider, value: LookupStatus::Skipped(reason), }; + if snapshot_failed { + pending.early.push(skip(SkipReason::SnapshotUnavailable)); + continue; + } let vendor_denied = registered .policy .tcf_vendor_id @@ -664,6 +704,14 @@ impl IdentityCoordinator for ProviderCoordinator

{ })); continue; } + if registered.policy.pull_sync + && self + .pull_sync_concurrency + .is_some_and(|max| pull_sync_dispatched >= max.get()) + { + pending.early.push(skip(SkipReason::RateLimited)); + continue; + } let input = LookupInput { ec_id: lookup.ec_id, consent: lookup.consent, @@ -723,6 +771,7 @@ impl IdentityCoordinator for ProviderCoordinator

{ let context = request.context.take(); match transport.start(request, deadline).await { Ok(value) => { + pull_sync_dispatched += usize::from(registered.policy.pull_sync); pending.requests.push(Tagged { provider, value }); if let Some(value) = context { pending.contexts.push(Tagged { provider, value }); @@ -1432,6 +1481,115 @@ mod tests { assert!(transport.sent.borrow().is_empty(), "should send nothing"); } + fn pull_sync_pair(max: Option) -> ProviderCoordinator { + let sources = ["a.example.com", "b.example.com"]; + let partners: Vec = sources + .iter() + .map(|source| { + serde_json::from_value(serde_json::json!({ + "name": source, "source_domain": source + })) + .expect("should build partner") + }) + .collect(); + let registered = sources + .iter() + .map(|source| { + let config: IdentityProviderConfig = toml::from_str(&format!( + "id = \"pull-sync:{source}\"\nkind = \"{PULL_SYNC_KIND}\"\nendpoint = \"https://ids.example.com/lookup\"\nallowed_domains = [\"ids.example.com\"]\nsources = [\"{source}\"]" + )) + .expect("should parse config"); + let mut provider = echo_provider(SourceInput::Anonymous); + provider.0.sources = Box::new([SourceCapability { + source: SourceDomain::try_from(*source).expect("should parse"), + input: SourceInput::Anonymous, + }]); + RegisteredProvider::new(config, provider) + }) + .collect(); + ProviderCoordinator::new( + registered, + Arc::new(PartnerRegistry::from_config(&partners).expect("should build registry")), + Duration::from_millis(200), + ) + .expect("should build coordinator") + .with_pull_sync_concurrency(max) + } + + #[test] + fn pull_sync_concurrency_caps_dispatched_lookups() { + let coordinator = pull_sync_pair(NonZeroUsize::new(1)); + let transport = CannedTransport::new(not_found_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + assert_eq!(transport.sent.borrow().len(), 1, "should send one lookup"); + let statuses: Vec = results.iter().map(|lookup| lookup.status).collect(); + assert_eq!( + statuses[1], + LookupStatus::Skipped(SkipReason::RateLimited), + "should skip the pull-sync lookup past the cap" + ); + } + + #[test] + fn pull_sync_refreshes_ownerless_legacy_uids() { + let coordinator = pull_sync_pair(None); + let transport = CannedTransport::new(not_found_reply); + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + for source in ["a.example.com", "b.example.com"] { + entry + .ids + .insert(source.to_owned(), KvPartnerId::single("legacy")); + } + + run( + &coordinator, + &transport, + &seeded_snapshot(&entry), + &granting(), + ); + + assert_eq!( + transport.sent.borrow().len(), + 2, + "should look up sources holding only ownerless legacy UIDs" + ); + } + + #[test] + fn pull_sync_concurrency_none_is_unlimited() { + let coordinator = pull_sync_pair(None); + let transport = CannedTransport::new(not_found_reply); + + run(&coordinator, &transport, &missing(), &granting()); + + assert_eq!(transport.sent.borrow().len(), 2, "should send both lookups"); + } + + #[test] + fn failed_snapshot_skips_every_provider() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(ok_reply); + + let results = run( + &coordinator, + &transport, + &EcKvSnapshot::Failed { + ec_id: EC_ID.to_owned(), + }, + &granting(), + ); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::SnapshotUnavailable), + "should skip when the snapshot read failed" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + #[test] fn tombstoned_row_skips_for_consent() { let coordinator = coordinator(""); @@ -1499,24 +1657,6 @@ mod tests { } } - #[test] - fn failed_snapshot_skips_every_provider() { - let coordinator = coordinator(""); - let transport = CannedTransport::new(ok_reply); - let snapshot = EcKvSnapshot::Failed { - ec_id: EC_ID.to_owned(), - }; - - let results = run(&coordinator, &transport, &snapshot, &granting()); - - assert_eq!( - only_status(&results), - LookupStatus::Skipped(SkipReason::NoRow), - "should skip when the row could not be read" - ); - assert!(transport.sent.borrow().is_empty(), "should send nothing"); - } - #[test] fn tcf_vendor_id_does_not_gate_traffic_without_tcf() { let coordinator = coordinator("tcf_vendor_id = 9999"); diff --git a/crates/trusted-server-core/src/ec/coordinator/results.rs b/crates/trusted-server-core/src/ec/coordinator/results.rs index 43a86103f8..9c3a90bb60 100644 --- a/crates/trusted-server-core/src/ec/coordinator/results.rs +++ b/crates/trusted-server-core/src/ec/coordinator/results.rs @@ -69,6 +69,9 @@ pub enum SkipReason { /// Every source of the provider is excluded where GDPR applies; the /// provider's stored UIDs are removed. Region, + /// The row snapshot read failed, so consent state (such as a withdrawal + /// tombstone) is unknown. + SnapshotUnavailable, } /// One outcome per provider per navigation. @@ -115,6 +118,7 @@ impl LookupStatus { Self::Skipped(SkipReason::Input) => "skipped_input", Self::Skipped(SkipReason::RateLimited) => "skipped_rate_limited", Self::Skipped(SkipReason::Region) => "skipped_region", + Self::Skipped(SkipReason::SnapshotUnavailable) => "skipped_snapshot_unavailable", Self::Skipped(SkipReason::Destination) => "destination_rejected", Self::Skipped(SkipReason::OptedOut) => "skipped_opted_out", } diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index cbc313a808..86d77fe927 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -76,13 +76,17 @@ pub fn ec_finalize_response( expire_retired_pull_marker(ec_context, response); } -/// Sets or expires the mirror cookie (spec §10). +/// Sets or expires the mirror cookie (spec §10). Best effort. /// /// Navigations rebuild it from the row as it stands after the page-load /// write: the committed row is stored as the `Present` snapshot, so /// [`super::EcKvSnapshot::entry_for`] returns it, or the read snapshot when no -/// write landed. Without EC and EID consent it is expired on navigations and on any -/// response whose request carried it. +/// write landed. Without EC and EID consent it is expired on any response +/// whose request carried it. +/// +/// The cookie is only expired when the request carried it, so responses to +/// browsers without a mirror stay free of `Set-Cookie`. When the row could +/// not be read authoritatively (KV failure), the mirror is left untouched. fn apply_mirror( ec_context: &EcContext, registry: &PartnerRegistry, @@ -92,9 +96,10 @@ fn apply_mirror( let decision = if !mirror_consent(ec_context.consent()) { MirrorCookie::Expire } else if navigation { - let row = ec_context - .ec_value() - .and_then(|ec_id| ec_context.kv_snapshot().entry_for(ec_id)); + let Some(row) = mirror_row(ec_context) else { + log::debug!("Leaving the mirror cookie untouched: identity row is unavailable"); + return; + }; build_mirror( row, ec_context.consent(), @@ -112,7 +117,7 @@ fn apply_mirror( value.as_str() ), ), - MirrorCookie::Expire if ec_context.mirror_cookie_present() || navigation => { + MirrorCookie::Expire if ec_context.mirror_cookie_present() => { append_cookie( response, &format!("{COOKIE_TS_EIDS_MIRROR}=; Path=/; Secure; SameSite=Lax; Max-Age=0"), @@ -122,6 +127,26 @@ fn apply_mirror( } } +/// The row the mirror is built from. +/// +/// Returns `Some(None)` when there is authoritatively no row (no EC or a +/// missing row), and `None` when the row is unknown because the read failed +/// or never happened. +fn mirror_row(ec_context: &EcContext) -> Option> { + let Some(ec_id) = ec_context.ec_value() else { + return Some(None); + }; + let snapshot = ec_context.kv_snapshot(); + match snapshot { + EcKvSnapshot::Present { .. } | EcKvSnapshot::Missing { .. } + if snapshot.belongs_to(ec_id) => + { + Some(snapshot.entry_for(ec_id)) + } + _ => None, + } +} + /// Expires a leftover pull-sync completeness marker; the marker is retired. fn expire_retired_pull_marker(ec_context: &EcContext, response: &mut Response) { if ec_context.pull_marker_present() { @@ -1980,7 +2005,11 @@ mod tests { ) } - fn run_mirror_navigation(lookups: LookupResults) -> Response { + fn run_mirror_navigation( + lookups: LookupResults, + mirror_present: bool, + kv_read_failed: bool, + ) -> Response { let settings = create_test_settings(); let ec_id = sample_ec_id("mirr01"); let graph = KvIdentityGraph::in_memory("mirror_store"); @@ -1993,7 +2022,15 @@ mod tests { graph .create(&ec_id, &live) .expect("should seed the live row"); - let mut ec_context = returning_user_context(&ec_id, graph.load_snapshot(&ec_id), true); + let snapshot = if kv_read_failed { + EcKvSnapshot::Failed { + ec_id: ec_id.clone(), + } + } else { + graph.load_snapshot(&ec_id) + }; + let mut ec_context = returning_user_context(&ec_id, snapshot, true); + ec_context.set_mirror_cookie_present_for_test(mirror_present); ec_context.set_lookup_results(lookups); let mut registry = PartnerRegistry::from_config(&[make_partner("ids.example.com")]) .expect("should build registry"); @@ -2014,7 +2051,7 @@ mod tests { #[test] fn navigation_sets_mirror_from_committed_row() { - let response = run_mirror_navigation(provider_found("ids.example.com")); + let response = run_mirror_navigation(provider_found("ids.example.com"), false, false); let cookies = mirror_cookies(&response); assert_eq!(cookies.len(), 1, "should set one mirror cookie"); @@ -2030,7 +2067,7 @@ mod tests { #[test] fn navigation_without_provider_uids_expires_mirror() { - let response = run_mirror_navigation(LookupResults::default()); + let response = run_mirror_navigation(LookupResults::default(), true, false); let cookies = mirror_cookies(&response); assert_eq!(cookies.len(), 1, "should emit one mirror cookie"); @@ -2040,6 +2077,54 @@ mod tests { ); } + #[test] + fn navigation_without_mirror_cookie_emits_no_expiry() { + let response = run_mirror_navigation(LookupResults::default(), false, false); + + assert!( + mirror_cookies(&response).is_empty(), + "should not expire a mirror the request did not carry" + ); + } + + #[test] + fn kv_read_failure_leaves_mirror_untouched() { + let response = run_mirror_navigation(provider_found("ids.example.com"), true, true); + + assert!( + mirror_cookies(&response).is_empty(), + "should leave the mirror untouched when the row could not be read" + ); + } + + #[test] + fn withdrawn_consent_without_mirror_cookie_emits_nothing() { + let settings = create_test_settings(); + let consent = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + source: ConsentSource::Cookie, + ..Default::default() + }; + let mut ec_context = make_context_with_consent(None, None, false, false, consent); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + None, + &PartnerRegistry::empty(), + None, + None, + &mut response, + ); + + assert!( + mirror_cookies(&response).is_empty(), + "should not expire a mirror the request did not carry" + ); + } + #[test] fn withdrawn_consent_expires_mirror_on_any_response() { let settings = create_test_settings(); diff --git a/crates/trusted-server-core/src/ec/mirror.rs b/crates/trusted-server-core/src/ec/mirror.rs index 72d678e05c..6cce5f003d 100644 --- a/crates/trusted-server-core/src/ec/mirror.rs +++ b/crates/trusted-server-core/src/ec/mirror.rs @@ -1,7 +1,7 @@ //! The JS-readable mirror of provider-owned bidstream UIDs (spec §10). use base64::Engine as _; -use serde_json::{Map, Value, json}; +use serde_json::{Map, Value}; use crate::consent::ConsentContext; use crate::consent::allows_eid_persistence; @@ -39,14 +39,15 @@ impl MirrorValue { /// Holds live, provider-written UIDs of `bidstream_enabled` partners, in /// partner config order, as unpadded base64url JSON /// `{"v":1,"eids":[...]}`. UIDs that would push the value past -/// [`MAX_MIRROR_BYTES`] are skipped. Returns [`MirrorCookie::Expire`] when -/// consent does not allow the EC or EID persistence, the row is missing or withdrawn, -/// or nothing qualifies. +/// [`MAX_MIRROR_BYTES`] are skipped, as are UIDs that fail to serialize. +/// Returns [`MirrorCookie::Expire`] when consent does not allow the EC or EID +/// persistence, the row is missing or withdrawn, or nothing qualifies. /// /// # Performance /// -/// Re-encodes once per kept UID; the size cap bounds this to a few dozen -/// small encodes per navigation. +/// Linear in the row size: each EID header and UID is serialized once, the +/// encoded length is tracked arithmetically, and the value is base64-encoded +/// once at the end. #[must_use] pub fn build_mirror( row: Option<&KvEntry>, @@ -57,8 +58,7 @@ pub fn build_mirror( let Some(row) = row.filter(|row| row.consent.ok && mirror_consent(consent)) else { return MirrorCookie::Expire; }; - let mut eids: Vec = Vec::new(); - let mut encoded: Option = None; + let mut mirror = MirrorBuilder::default(); for partner in partners .in_config_order() .filter(|partner| partner.bidstream_enabled) @@ -66,50 +66,150 @@ pub fn build_mirror( let Some(record) = row.ids.get(&partner.source_domain) else { continue; }; + let source = partner.source_domain.as_str(); + // Ownerless UIDs of a pull-sync partner's source were written by + // legacy pull sync, so they belong to that partner's provider. + let pull_sync_owned = + partner.provider_owned && partner.pull_sync_enabled && partner.pull_sync_url.is_some(); + if record.eids().is_empty() { + if pull_sync_owned && !record.uid.is_empty() { + let header = legacy_header(source); + let uid = serde_json::to_string(&serde_json::json!({ + "id": record.uid, + "atype": partner.openrtb_atype, + })); + mirror.push_eid(source, header, [uid]); + } + continue; + } for eid in record.eids() { - let entry_index = eids.len(); - eids.push(eid_json(&partner.source_domain, eid)); - let mut kept = 0_usize; - for uid in eid + let uids = eid .uids() .iter() - .filter(|uid| uid.owner().is_some() && uid.is_live(now)) - { - uids_mut(&mut eids[entry_index]).push(uid_json(uid)); - if let Some(value) = encode(&eids) { - encoded = Some(value); - kept += 1; - } else { - uids_mut(&mut eids[entry_index]).pop(); - } + .filter(|uid| (pull_sync_owned || uid.owner().is_some()) && uid.is_live(now)) + .map(|uid| serde_json::to_string(&uid_json(uid))); + mirror.push_eid(source, eid_header(source, eid), uids); + } + } + let MirrorBuilder { + mut json, + eid_count, + } = mirror; + if eid_count == 0 { + return MirrorCookie::Expire; + } + json.push_str(JSON_SUFFIX); + let value = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(json); + debug_assert!( + value.len() <= MAX_MIRROR_BYTES, + "should respect the size cap" + ); + MirrorCookie::Set(MirrorValue(value)) +} + +/// Accumulates the mirror JSON, tracking its encoded size so each part is +/// serialized once. +struct MirrorBuilder { + json: String, + eid_count: usize, +} + +impl Default for MirrorBuilder { + fn default() -> Self { + Self { + json: String::from(JSON_PREFIX), + eid_count: 0, + } + } +} + +impl MirrorBuilder { + /// Appends one EID with every UID that still fits; drops the EID when + /// none does. Parts that fail to serialize are skipped and logged. + fn push_eid( + &mut self, + source: &str, + header: Option, + uids: impl IntoIterator>, + ) { + let Some(header) = header else { + log::warn!("Skipping mirror EID for '{source}': header does not serialize"); + return; + }; + let eid_start = self.json.len(); + let mut kept = 0_usize; + for uid in uids { + let Ok(uid) = uid else { + log::warn!("Skipping mirror UID for '{source}': does not serialize"); + continue; + }; + let added = if kept == 0 { + usize::from(self.eid_count > 0) + header.len() + uid.len() + EID_SUFFIX.len() + } else { + 1 + uid.len() + }; + if encoded_len(self.json.len() + added + JSON_SUFFIX.len()) > MAX_MIRROR_BYTES { + continue; } if kept == 0 { - eids.pop(); + if self.eid_count > 0 { + self.json.push(','); + } + self.json.push_str(&header); + } else { + self.json.push(','); } + self.json.push_str(&uid); + kept += 1; + } + if kept > 0 { + self.json.push_str(EID_SUFFIX); + self.eid_count += 1; + } else { + self.json.truncate(eid_start); } - } - match encoded { - Some(value) if !eids.is_empty() => MirrorCookie::Set(MirrorValue(value)), - _ => MirrorCookie::Expire, } } -/// Whether consent allows the mirror: EC consent and EID persistence. -#[must_use] -pub(crate) fn mirror_consent(consent: &ConsentContext) -> bool { - ec_consent_granted(consent) && allows_eid_persistence(consent) +/// Header for a legacy single-UID record: the source only. +fn legacy_header(source: &str) -> Option { + let mut object = Map::new(); + object.insert("source".to_owned(), Value::from(source)); + object.insert("uids".to_owned(), Value::Array(Vec::new())); + open_uids_array(serde_json::to_string(&Value::Object(object)).ok()?) } -fn uids_mut(eid: &mut Value) -> &mut Vec { - eid["uids"] - .as_array_mut() - .expect("should hold a uids array") +const JSON_PREFIX: &str = r#"{"v":1,"eids":["#; +const JSON_SUFFIX: &str = "]}"; +const EID_SUFFIX: &str = "]}"; + +/// Unpadded base64 length of `bytes` input bytes. +fn encoded_len(bytes: usize) -> usize { + (bytes * 4).div_ceil(3) } -fn encode(eids: &[Value]) -> Option { - let json = serde_json::to_vec(&json!({ "v": 1, "eids": eids })).ok()?; - let value = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(json); - (value.len() <= MAX_MIRROR_BYTES).then_some(value) +/// Serializes an EID without its UIDs, ending in an open `"uids":[`. +/// +/// `uids` sorts last among the EID's keys, so the empty array closes the +/// serialized object. +fn eid_header(source: &str, eid: &KvEid) -> Option { + open_uids_array(serde_json::to_string(&eid_json(source, eid)).ok()?) +} + +/// Turns a serialized EID ending in `"uids":[]}` into one ending in an open +/// `"uids":[`. +fn open_uids_array(mut header: String) -> Option { + let open = header.strip_suffix(EID_SUFFIX)?.len(); + header.ends_with(r#""uids":[]}"#).then(|| { + header.truncate(open); + header + }) +} + +/// Whether consent allows the mirror: EC consent and EID persistence. +#[must_use] +pub(crate) fn mirror_consent(consent: &ConsentContext) -> bool { + ec_consent_granted(consent) && allows_eid_persistence(consent) } fn eid_json(source: &str, eid: &KvEid) -> Value { @@ -150,6 +250,7 @@ mod tests { use super::*; use crate::ec::eid_types::{AgentType, EidFields, ProviderId, UidValue}; use crate::ec::kv_types::KvPartnerId; + use serde_json::json; fn partners(bidstream: bool) -> PartnerRegistry { let partner = serde_json::from_value(json!({ @@ -251,6 +352,73 @@ mod tests { ); } + #[test] + fn mirrors_legacy_uid_of_pull_sync_partner_only() { + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry + .ids + .insert("ids.example.com".to_owned(), KvPartnerId::single("legacy")); + let pull_partner = serde_json::from_value(json!({ + "name": "ids", "source_domain": "ids.example.com", "bidstream_enabled": true, + "pull_sync_enabled": true, "pull_sync_url": "https://sync.example.com/pull", + "pull_sync_allowed_domains": ["sync.example.com"], + "ts_pull_token": "example-pull-token-32-bytes-minimum" + })) + .expect("should build partner"); + let mut pull = + PartnerRegistry::from_config(&[pull_partner]).expect("should build registry"); + pull.mark_provider_owned("ids.example.com", None); + let now = UnixSeconds::new(1000); + + let MirrorCookie::Set(value) = build_mirror(Some(&entry), &granting(), &pull, now) else { + panic!("should mirror the legacy pull-sync UID"); + }; + + assert_eq!( + decode(&value)["eids"][0]["uids"][0]["id"], + json!("legacy"), + "should attribute the ownerless UID to the pull-sync partner" + ); + assert_eq!( + build_mirror(Some(&entry), &granting(), &partners(true), now), + MirrorCookie::Expire, + "should not mirror ownerless UIDs of other partners" + ); + } + + #[test] + fn mirrors_several_eids_as_valid_json() { + let mut entry = row(vec![uid("one", Expiry::Never), uid("two", Expiry::Never)]); + let record = KvPartnerId::from_eids( + vec![ + KvEid::new(EidFields::default(), vec![uid("a", Expiry::Never)]), + KvEid::new(EidFields::default(), vec![uid("b", Expiry::Never)]), + ], + UnixSeconds::new(0), + ) + .expect("should build record"); + entry.ids.insert("ids.example.com".to_owned(), record); + + let MirrorCookie::Set(value) = build_mirror( + Some(&entry), + &granting(), + &partners(true), + UnixSeconds::new(1000), + ) else { + panic!("should set the mirror"); + }; + + assert_eq!( + decode(&value), + json!({ "v": 1, "eids": [ + { "source": "ids.example.com", "uids": [{ "id": "a", "atype": 1 }] }, + { "source": "ids.example.com", "uids": [{ "id": "b", "atype": 1 }] } + ] }), + "should join EIDs into valid JSON" + ); + } + #[test] fn skips_uids_that_do_not_fit() { let big: Vec = (0..20) diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs index e942de30bb..3ebddd1ed4 100644 --- a/crates/trusted-server-core/src/ec/provider.rs +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -680,11 +680,12 @@ pub(crate) fn parse_sources( } /// Whether `url` is `https` and its host is one of `allowed_domains`, -/// compared ASCII case-insensitively. +/// compared ASCII case-insensitively, ignoring a trailing dot on the host. #[must_use] pub(crate) fn destination_allowed>(url: &Url, allowed_domains: &[S]) -> bool { url.scheme() == "https" && url.host_str().is_some_and(|host| { + let host = host.strip_suffix('.').unwrap_or(host); allowed_domains .iter() .any(|allowed| allowed.as_ref().eq_ignore_ascii_case(host)) diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 72f5a4a7be..30513ff1c7 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -761,8 +761,8 @@ pub struct Ec { #[serde(default)] pub ec_store: Option, - /// Maximum number of concurrent pull-sync requests; retained for config - /// compatibility; unused. + /// Maximum number of pull-sync lookups dispatched per navigation; 0 + /// means no limit. #[serde(default = "Ec::default_pull_sync_concurrency")] pub pull_sync_concurrency: usize, @@ -855,6 +855,19 @@ impl Ec { "replace-with-random-ec-passphrase", ]; + /// Normalizes every partner's `pull_sync_allowed_domains`: trimmed, + /// lowercased and without a trailing dot, matching how request hosts are + /// compared. + fn normalize_pull_sync_domains(&mut self) { + for domain in self + .partners + .iter_mut() + .flat_map(|partner| partner.pull_sync_allowed_domains.iter_mut()) + { + *domain = domain.trim().trim_end_matches('.').to_ascii_lowercase(); + } + } + /// Default maximum concurrent pull-sync requests. #[must_use] pub const fn default_pull_sync_concurrency() -> usize { @@ -3618,6 +3631,7 @@ impl Settings { self.image_optimizer.normalize()?; self.debug.auction_html_comment_options.normalize(); self.tinybird.normalize(); + self.ec.normalize_pull_sync_domains(); self.integrations .remove_legacy_static_secret_store_selectors(); self.consent.validate(); @@ -4562,6 +4576,30 @@ mod tests { ); } + #[test] + fn pull_sync_allowed_domains_are_normalized_at_load() { + let toml = format!( + r#"{} + [[ec.partners]] + name = "Pull partner" + source_domain = "partner.example.com" + pull_sync_enabled = true + pull_sync_url = "https://sync.example.com/pull" + pull_sync_allowed_domains = [" Sync.Example.COM. "] + ts_pull_token = "example-pull-token-32-bytes-minimum" + "#, + crate_test_settings_str() + ); + + let settings = Settings::from_toml(&toml).expect("should parse settings"); + + assert_eq!( + settings.ec.partners[0].pull_sync_allowed_domains, + vec!["sync.example.com".to_owned()], + "should trim, lowercase and drop the trailing dot" + ); + } + #[test] fn html_finalization_timeout_defaults_and_boundaries() { let parsed = Settings::from_toml(&crate_test_settings_str()) diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index 69256bdad1..7cda004904 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -22,7 +22,7 @@ pub mod sharedid; pub mod uid2; pub mod unifiedid; -use std::num::NonZeroU32; +use std::num::{NonZeroU32, NonZeroUsize}; use std::sync::Arc; use std::time::Duration; @@ -273,7 +273,9 @@ fn build_phase_coordinator( if registered.is_empty() { return Ok(None); } - ProviderCoordinator::new(registered, partners, budget).map(Some) + let pull_sync_concurrency = NonZeroUsize::new(settings.ec.pull_sync_concurrency); + ProviderCoordinator::new(registered, partners, budget) + .map(|coordinator| Some(coordinator.with_pull_sync_concurrency(pull_sync_concurrency))) } #[cfg(test)] diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 65f214f6dd..f01031f4d5 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -863,7 +863,7 @@ be at least 32 bytes. Keep it stable to preserve EC identifier continuity. | ----------------------------------- | -------------- | -------- | ----------------------------------------------------------------------- | | `passphrase` | String | Yes | Publisher passphrase used as HMAC key | | `ec_store` | String or null | No | Fastly KV store name for EC identity graph and withdrawal state | -| `pull_sync_concurrency` | Integer | No | Maximum concurrent pull-sync requests per organic response | +| `pull_sync_concurrency` | Integer | No | Maximum pull-sync lookups dispatched per navigation (0 = no limit) | | `cluster_trust_threshold` | Integer | No | Cluster size threshold for identity trust decisions | | `cluster_recheck_secs` | Integer | No | Legacy compatibility setting; cluster rechecks no longer use timestamps | | `partners` | Array | No | Static partner registry entries | diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index 14781bd465..3ad7c71f8c 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -278,7 +278,8 @@ Trusted Server mirrors provider-written UIDs for `bidstream_enabled` partners in - **Name**: `__Host-ts-eids-mirror` - **Attributes**: `Path=/; Secure; SameSite=Lax; Max-Age=`, host-only (no `Domain`) and not `HttpOnly` - **Contents**: unpadded base64url JSON `{"v":1,"eids":[{"source":"ids.example.com","uids":[{"id":"...","atype":1,"exp":1767225600}]}]}`, partners in `[[ec.partners]]` order, at most 3584 bytes; UIDs that do not fit are left out -- **Updates**: every page navigation rebuilds it from the row after that navigation's write, and expires it when no live provider UID qualifies. Without EC and EID consent it is expired on navigations and on any response whose request carried it. +- **Contents source**: live UIDs written by an identity provider. UIDs that legacy pull sync stored without an owner count as the pull-sync partner's own. +- **Updates** (best effort): page navigations rebuild it from the row after that navigation's write. It is expired only when the request carried it, either because no live provider UID qualifies or because EC and EID consent is missing (on any response). Responses to browsers without a mirror get no mirror `Set-Cookie`. When the identity row cannot be read (KV failure) the mirror is left untouched, and a mirror that cannot be built is skipped without failing the response. Readers must drop UIDs whose `exp` (Unix seconds) has passed and strip `exp` before using a UID. From 07a96eea125bdd5a11d27655cb66b0c65b521c9b Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Tue, 3 Nov 2026 20:09:23 +0530 Subject: [PATCH 04/20] Stop mirroring ownerless UIDs and re-pulling held pull-sync sources The mirror cookie now holds only UIDs with a provider owner, so browser, batch and legacy pull-sync records are never mirrored, and its size budget counts the pending EID suffix for later UIDs. A pull-sync source with any live UID counts as held, as legacy pull sync did, so ownerless records no longer send the EC ID to the partner on every navigation. Config load warns when a pull-sync partner also sets a batch api_token, whose mappings batch sync rejects. Restore a test that an oversized pull-sync UID is dropped, and correct the guide on browser fallback ingest. Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 80 +++++++++++++------ crates/trusted-server-core/src/ec/mirror.rs | 80 ++++++++++--------- crates/trusted-server-core/src/ec/registry.rs | 48 ++++++++++- docs/guide/ec-setup-guide.md | 7 ++ docs/guide/edge-cookies.md | 4 +- 5 files changed, 150 insertions(+), 69 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 9b875f20b6..be66a34eab 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -310,7 +310,7 @@ impl ProviderCoordinator

{ } /// Claimed sources lacking a live provider-written UID (a browser - /// fallback does not count; for pull sync, a live owned UID), minus + /// fallback does not count; for pull sync, any live UID counts), minus /// sources needing input V1 /// never has; the flag reports whether any source was removed for input. fn due_sources( @@ -332,19 +332,14 @@ impl ProviderCoordinator

{ due.push(source.clone()); continue; } - // Legacy pull sync wrote single-UID records with no owner; a - // pull-sync provider treats them as its own and refreshes them so - // they gain an owner and are mirrored. + // A pull-sync source counts any live UID, owned or not, as held: + // like legacy pull sync, it pulls only for sources with no UID, so + // an ownerless browser, batch or legacy record does not trigger a + // lookup (and an EC ID send) on every navigation. Sources with no + // UID stay bounded by the partner's hourly pull rate limit. let held = record.is_some_and(|record| { - if registered.policy.pull_sync { - record - .eids() - .iter() - .flat_map(KvEid::uids) - .any(|uid| uid.owner().is_some() && uid.is_live(now)) - } else { - record.is_provider_written() && record.has_live_uid(now) - } + (registered.policy.pull_sync || record.is_provider_written()) + && record.has_live_uid(now) }); if held { continue; @@ -1533,29 +1528,62 @@ mod tests { } #[test] - fn pull_sync_refreshes_ownerless_legacy_uids() { + fn pull_sync_holds_ownerless_uids_without_repulling() { let coordinator = pull_sync_pair(None); let transport = CannedTransport::new(not_found_reply); let mut entry = KvEntry::tombstone(1000); entry.consent.ok = true; - for source in ["a.example.com", "b.example.com"] { - entry - .ids - .insert(source.to_owned(), KvPartnerId::single("legacy")); - } + entry + .ids + .insert("a.example.com".to_owned(), KvPartnerId::single("legacy")); + let snapshot = seeded_snapshot(&entry); - run( - &coordinator, - &transport, - &seeded_snapshot(&entry), - &granting(), - ); + for _ in 0..2 { + run(&coordinator, &transport, &snapshot, &granting()); + } assert_eq!( transport.sent.borrow().len(), 2, - "should look up sources holding only ownerless legacy UIDs" + "should look up only the empty source, once per navigation, after a no-match" + ); + assert!( + transport + .sent + .borrow() + .iter() + .all(|url| url.ends_with("due=1")), + "should never include the ownerless source" + ); + } + + #[test] + fn pull_sync_drops_oversized_uid() { + let config: IdentityProviderConfig = toml::from_str(&format!( + "id = \"pull-sync:ids.example.com\"\nkind = \"{PULL_SYNC_KIND}\"\nendpoint = \"https://ids.example.com/lookup\"\nallowed_domains = [\"ids.example.com\"]\nsources = [\"ids.example.com\"]" + )) + .expect("should parse config"); + let coordinator = coordinator_with(RegisteredProvider::new( + config, + echo_provider(SourceInput::Anonymous), + )); + let transport = CannedTransport::new(|| { + Ok(TransportResponse { + status: http::StatusCode::OK, + headers: http::HeaderMap::new(), + body: bytes::Bytes::from("x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1)), + }) + }); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + let lookup = results.iter().next().expect("should report the provider"); + assert_eq!( + lookup.status, + LookupStatus::NoMatch, + "should treat a pull with only an oversized UID as no match" ); + assert!(lookup.entries.is_empty(), "should store no oversized UID"); } #[test] diff --git a/crates/trusted-server-core/src/ec/mirror.rs b/crates/trusted-server-core/src/ec/mirror.rs index 6cce5f003d..f26b2f41cb 100644 --- a/crates/trusted-server-core/src/ec/mirror.rs +++ b/crates/trusted-server-core/src/ec/mirror.rs @@ -67,26 +67,14 @@ pub fn build_mirror( continue; }; let source = partner.source_domain.as_str(); - // Ownerless UIDs of a pull-sync partner's source were written by - // legacy pull sync, so they belong to that partner's provider. - let pull_sync_owned = - partner.provider_owned && partner.pull_sync_enabled && partner.pull_sync_url.is_some(); - if record.eids().is_empty() { - if pull_sync_owned && !record.uid.is_empty() { - let header = legacy_header(source); - let uid = serde_json::to_string(&serde_json::json!({ - "id": record.uid, - "atype": partner.openrtb_atype, - })); - mirror.push_eid(source, header, [uid]); - } - continue; - } + // Only provider-written UIDs carry an owner. Ownerless records come + // from browser ingest, batch sync or legacy pull sync, so they are + // never mirrored. for eid in record.eids() { let uids = eid .uids() .iter() - .filter(|uid| (pull_sync_owned || uid.owner().is_some()) && uid.is_live(now)) + .filter(|uid| uid.owner().is_some() && uid.is_live(now)) .map(|uid| serde_json::to_string(&uid_json(uid))); mirror.push_eid(source, eid_header(source, eid), uids); } @@ -146,7 +134,7 @@ impl MirrorBuilder { let added = if kept == 0 { usize::from(self.eid_count > 0) + header.len() + uid.len() + EID_SUFFIX.len() } else { - 1 + uid.len() + 1 + uid.len() + EID_SUFFIX.len() }; if encoded_len(self.json.len() + added + JSON_SUFFIX.len()) > MAX_MIRROR_BYTES { continue; @@ -171,14 +159,6 @@ impl MirrorBuilder { } } -/// Header for a legacy single-UID record: the source only. -fn legacy_header(source: &str) -> Option { - let mut object = Map::new(); - object.insert("source".to_owned(), Value::from(source)); - object.insert("uids".to_owned(), Value::Array(Vec::new())); - open_uids_array(serde_json::to_string(&Value::Object(object)).ok()?) -} - const JSON_PREFIX: &str = r#"{"v":1,"eids":["#; const JSON_SUFFIX: &str = "]}"; const EID_SUFFIX: &str = "]}"; @@ -353,12 +333,12 @@ mod tests { } #[test] - fn mirrors_legacy_uid_of_pull_sync_partner_only() { + fn does_not_mirror_browser_ingested_uid_of_pull_sync_source() { let mut entry = KvEntry::tombstone(1000); entry.consent.ok = true; entry .ids - .insert("ids.example.com".to_owned(), KvPartnerId::single("legacy")); + .insert("ids.example.com".to_owned(), KvPartnerId::single("browser")); let pull_partner = serde_json::from_value(json!({ "name": "ids", "source_domain": "ids.example.com", "bidstream_enabled": true, "pull_sync_enabled": true, "pull_sync_url": "https://sync.example.com/pull", @@ -369,21 +349,13 @@ mod tests { let mut pull = PartnerRegistry::from_config(&[pull_partner]).expect("should build registry"); pull.mark_provider_owned("ids.example.com", None); - let now = UnixSeconds::new(1000); - let MirrorCookie::Set(value) = build_mirror(Some(&entry), &granting(), &pull, now) else { - panic!("should mirror the legacy pull-sync UID"); - }; + let mirror = build_mirror(Some(&entry), &granting(), &pull, UnixSeconds::new(1000)); assert_eq!( - decode(&value)["eids"][0]["uids"][0]["id"], - json!("legacy"), - "should attribute the ownerless UID to the pull-sync partner" - ); - assert_eq!( - build_mirror(Some(&entry), &granting(), &partners(true), now), + mirror, MirrorCookie::Expire, - "should not mirror ownerless UIDs of other partners" + "should not mirror an ownerless browser-ingested UID" ); } @@ -445,4 +417,36 @@ mod tests { .len(); assert!(kept > 0 && kept < 20, "should keep a prefix that fits"); } + + #[test] + fn several_uids_of_one_eid_respect_the_cap_at_every_boundary() { + // Three fixed UIDs fill most of the budget; growing the last UID one + // byte at a time crosses the cap at every possible offset. + for tail in 1..400 { + let mut uids: Vec = (0..3) + .map(|i| uid(&format!("{i}{}", "x".repeat(800)), Expiry::Never)) + .collect(); + uids.push(uid(&"y".repeat(tail), Expiry::Never)); + let entry = row(uids); + + let MirrorCookie::Set(value) = build_mirror( + Some(&entry), + &granting(), + &partners(true), + UnixSeconds::new(1000), + ) else { + panic!("should set the mirror"); + }; + + assert!( + value.as_str().len() <= MAX_MIRROR_BYTES, + "should fit the size limit with tail {tail}" + ); + let kept = decode(&value)["eids"][0]["uids"] + .as_array() + .expect("should hold uids") + .len(); + assert!(kept >= 3, "should keep the fixed UIDs with tail {tail}"); + } + } } diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index 928afcb0f9..265e784ec9 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -296,9 +296,18 @@ impl PartnerRegistry { } } let pull_sync_registered = is_registered_kind(PULL_SYNC_KIND); - for partner in ec.partners.iter().filter(|partner| { - pull_sync_registered && partner.pull_sync_enabled && partner.pull_sync_url.is_some() - }) { + for partner in ec + .partners + .iter() + .filter(|partner| pull_sync_registered && runs_pull_sync(partner)) + { + if batch_sync_rejected(partner) { + log::warn!( + "Partner '{}' runs pull sync and has a batch sync api_token; its source is \ + provider-owned, so batch sync rejects every mapping for it", + partner.source_domain + ); + } registry.mark_provider_owned(&partner.source_domain, None); } Ok(registry) @@ -425,6 +434,18 @@ impl PartnerRegistry { } } +/// Whether the partner runs pull sync as a provider, making its source +/// provider-owned. +fn runs_pull_sync(partner: &EcPartner) -> bool { + partner.pull_sync_enabled && partner.pull_sync_url.is_some() +} + +/// Whether a pull-sync partner also has a batch `api_token`, whose mappings +/// batch sync then rejects because the source is provider-owned. +fn batch_sync_rejected(partner: &EcPartner) -> bool { + runs_pull_sync(partner) && partner.api_token.is_some() +} + fn validate_api_token( source_domain: &str, api_token: &str, @@ -706,6 +727,27 @@ mod tests { ); } + #[test] + fn batch_token_on_pull_sync_partner_is_flagged() { + let mut pull = make_partner("pull.example.com", &valid_api_token("pull-partner")); + + assert!( + !batch_sync_rejected(&pull), + "should not flag a batch-only partner" + ); + pull.pull_sync_enabled = true; + pull.pull_sync_url = Some("https://sync.example.com/pull".to_owned()); + assert!( + batch_sync_rejected(&pull), + "should flag a pull-sync partner with a batch token" + ); + pull.api_token = None; + assert!( + !batch_sync_rejected(&pull), + "should not flag a pull-sync partner without a batch token" + ); + } + #[test] fn from_ec_with_kinds_rejects_an_unregistered_kind() { let mut ec = crate::test_support::tests::create_test_settings().ec; diff --git a/docs/guide/ec-setup-guide.md b/docs/guide/ec-setup-guide.md index cccb04daee..16207049c9 100644 --- a/docs/guide/ec-setup-guide.md +++ b/docs/guide/ec-setup-guide.md @@ -42,6 +42,13 @@ do not call either API may omit it. Provision high-entropy values under `ec_passphrase` and `partner_api_token`; see [Configuration](/guide/configuration#secret-store-migration). +A partner with pull sync enabled (`pull_sync_enabled` and `pull_sync_url`) +owns its source through the pull-sync provider, so batch sync rejects every +mapping for it with `provider_owned`. Trusted Server logs a warning at config +load when such a partner also sets `api_token`; use either pull sync or batch +sync for a partner, not both. See +[Pull Sync on Navigations](/guide/edge-cookies#pull-sync-on-navigations). + Required behavior assumptions: - The value stored under `ec_passphrase` is long-lived HMAC-SHA256 keying material for EC ID derivation; use a high-entropy random value of at least 32 characters diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index 3ad7c71f8c..b4f2fad29c 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -265,7 +265,7 @@ Server-resolved EIDs and current-request Prebid EIDs are deduplicated by `source ### Pull Sync on Navigations -Pull sync runs as an identity provider during top-level page navigations, within the shared identity lookup budget, and its result lands in the navigation's single KV write. A pull-sync partner's source is provider-owned: browser EID ingestion skips it and batch sync rejects its mappings with `provider_owned`. +Pull sync runs as an identity provider during top-level page navigations, within the shared identity lookup budget, and its result lands in the navigation's single KV write. A pull-sync partner's source is provider-owned: browser EID ingestion stores a fallback UID for it only when the row holds no live UID for that source, and batch sync rejects its mappings with `provider_owned`. Trusted Server logs a warning at config load when a pull-sync partner also has a batch `api_token`, since that token's mappings are all rejected. A source that already holds any live UID, including an ownerless fallback, is not pulled again; a source with no UID is pulled at most `pull_sync_rate_limit` times per hour per EC. The former `ts-ec-pull-complete` completeness marker is retired. A response to a request that still carries it expires the leftover cookie. @@ -278,7 +278,7 @@ Trusted Server mirrors provider-written UIDs for `bidstream_enabled` partners in - **Name**: `__Host-ts-eids-mirror` - **Attributes**: `Path=/; Secure; SameSite=Lax; Max-Age=`, host-only (no `Domain`) and not `HttpOnly` - **Contents**: unpadded base64url JSON `{"v":1,"eids":[{"source":"ids.example.com","uids":[{"id":"...","atype":1,"exp":1767225600}]}]}`, partners in `[[ec.partners]]` order, at most 3584 bytes; UIDs that do not fit are left out -- **Contents source**: live UIDs written by an identity provider. UIDs that legacy pull sync stored without an owner count as the pull-sync partner's own. +- **Contents source**: live UIDs written by an identity provider. Ownerless UIDs, from browser ingest, batch sync or legacy pull sync, are never mirrored. - **Updates** (best effort): page navigations rebuild it from the row after that navigation's write. It is expired only when the request carried it, either because no live provider UID qualifies or because EC and EID consent is missing (on any response). Responses to browsers without a mirror get no mirror `Set-Cookie`. When the identity row cannot be read (KV failure) the mirror is left untouched, and a mirror that cannot be built is skipped without failing the response. Readers must drop UIDs whose `exp` (Unix seconds) has passed and strip `exp` before using a UID. From 99895ae6ab8570af36c2ccb2ac88506024e7f23f Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 20:09:49 +0530 Subject: [PATCH 05/20] Keep batch sync for pull-sync partners Batch sync rejects only sources owned by a configured identity provider. A source owned only by its partner's pull sync accepts batch mappings again, as it did before pull sync became a provider. Signed-off-by: dhruv8sh --- .../trusted-server-core/src/ec/batch_sync.rs | 71 +++++++++++++++++- crates/trusted-server-core/src/ec/registry.rs | 75 ++++++++++--------- docs/guide/configuration.md | 2 +- docs/guide/ec-setup-guide.md | 6 +- docs/guide/edge-cookies.md | 2 +- 5 files changed, 114 insertions(+), 42 deletions(-) diff --git a/crates/trusted-server-core/src/ec/batch_sync.rs b/crates/trusted-server-core/src/ec/batch_sync.rs index 756cc0f490..e9b2b319e0 100644 --- a/crates/trusted-server-core/src/ec/batch_sync.rs +++ b/crates/trusted-server-core/src/ec/batch_sync.rs @@ -172,7 +172,7 @@ fn handle_batch_sync_with_writer( let (accepted, errors) = process_mappings_for_partner( writer, &partner.source_domain, - registry.is_provider_owned(&partner.source_domain), + registry.rejects_batch_sync(&partner.source_domain), &body.mappings, ); @@ -992,6 +992,75 @@ mod tests { ); } + #[test] + fn handle_batch_sync_accepts_mappings_for_a_pull_sync_owned_source() { + // Arrange: pull sync owns the source, but no configured provider does. + let mut registry = test_registry(); + assert!( + registry.mark_provider_owned("ssp.example.com", None), + "should mark the pull-sync source" + ); + let limiter = MockRateLimiter { + should_exceed: false, + }; + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let writer = MockWriter::new(vec![Ok(UpsertResult::Written)]); + let body = + format!(r#"{{"mappings":[{{"ec_id":"{ec_id}","partner_uid":"one","timestamp":1}}]}}"#); + + // Act + let response = handle_batch_sync_with_writer( + &writer, + ®istry, + &limiter, + authorized_batch_request(&body), + ) + .expect("should return success response"); + + // Assert + assert_eq!( + response.status(), + StatusCode::OK, + "should accept batch mappings for a pull-sync partner" + ); + assert_eq!( + response_json(response)["accepted"], + 1, + "should accept the mapping" + ); + } + + #[test] + fn handle_batch_sync_rejects_mappings_for_a_configured_provider_source() { + let mut registry = test_registry(); + assert!( + registry.reject_batch_sync("ssp.example.com"), + "should mark the configured provider source" + ); + let limiter = MockRateLimiter { + should_exceed: false, + }; + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let writer = MockWriter::new(vec![Ok(UpsertResult::Written)]); + let body = + format!(r#"{{"mappings":[{{"ec_id":"{ec_id}","partner_uid":"one","timestamp":1}}]}}"#); + + let response = handle_batch_sync_with_writer( + &writer, + ®istry, + &limiter, + authorized_batch_request(&body), + ) + .expect("should return response"); + + let json = response_json(response); + assert_eq!(json["accepted"], 0, "should accept nothing"); + assert_eq!( + json["errors"][0]["reason"], REASON_PROVIDER_OWNED, + "should report provider_owned" + ); + } + #[test] fn handle_batch_sync_accepts_every_member_of_a_written_group() { let registry = test_registry(); diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index 265e784ec9..f9aa07ad8f 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -54,7 +54,7 @@ pub struct PartnerConfig { /// Outbound bearer token for pull sync requests. pub ts_pull_token: Option>, /// Whether an enabled identity provider claims this source. Browser - /// ingest skips it and batch sync rejects it. + /// ingest skips it. pub provider_owned: bool, /// TCF vendor IDs of the configured providers, enabled or disabled, that /// claim this source. Its UIDs leave the server only while every one of @@ -64,6 +64,11 @@ pub struct PartnerConfig { /// source where GDPR applies. Its UIDs then never leave the server for a /// visitor whose jurisdiction is GDPR. pub excluded_where_gdpr_applies: bool, + /// Whether batch sync rejects this source's mappings: an enabled + /// configured provider owns it. A source owned only by its partner's pull + /// sync keeps accepting batch mappings, as it did before pull sync became + /// a provider. + pub batch_sync_rejected: bool, } impl PartnerConfig { @@ -286,6 +291,7 @@ impl PartnerRegistry { } for source in &provider.sources { registry.mark_provider_owned(source, provider.tcf_vendor_id); + registry.reject_batch_sync(source); } } // A disabled provider's stored UIDs stay behind its vendor consent, @@ -301,13 +307,6 @@ impl PartnerRegistry { .iter() .filter(|partner| pull_sync_registered && runs_pull_sync(partner)) { - if batch_sync_rejected(partner) { - log::warn!( - "Partner '{}' runs pull sync and has a batch sync api_token; its source is \ - provider-owned, so batch sync rejects every mapping for it", - partner.source_domain - ); - } registry.mark_provider_owned(&partner.source_domain, None); } Ok(registry) @@ -397,6 +396,30 @@ impl PartnerRegistry { .is_some_and(|partner| partner.provider_owned) } + /// Marks `source` as owned by an enabled configured provider, so batch + /// sync rejects its mappings. Returns whether `source` is registered. + pub fn reject_batch_sync(&mut self, source: &str) -> bool { + let Ok(normalized) = normalize_partner_source_domain(source) else { + return false; + }; + match self.by_source_domain.get_mut(&normalized) { + Some(partner) => { + partner.batch_sync_rejected = true; + true + } + None => false, + } + } + + /// Whether batch sync rejects mappings for `source` (spec §8.5): an + /// enabled configured provider owns it. Pull-sync ownership alone does + /// not reject batch mappings. + #[must_use] + pub fn rejects_batch_sync(&self, source: &str) -> bool { + self.find_by_source_domain(source) + .is_some_and(|partner| partner.batch_sync_rejected) + } + /// Returns all partners with `pull_sync_enabled = true`. #[must_use] pub fn pull_enabled_partners(&self) -> Vec<&PartnerConfig> { @@ -440,12 +463,6 @@ fn runs_pull_sync(partner: &EcPartner) -> bool { partner.pull_sync_enabled && partner.pull_sync_url.is_some() } -/// Whether a pull-sync partner also has a batch `api_token`, whose mappings -/// batch sync then rejects because the source is provider-owned. -fn batch_sync_rejected(partner: &EcPartner) -> bool { - runs_pull_sync(partner) && partner.api_token.is_some() -} - fn validate_api_token( source_domain: &str, api_token: &str, @@ -490,6 +507,7 @@ fn build_partner_config( provider_owned: false, owner_tcf_vendor_ids: Vec::new(), excluded_where_gdpr_applies: false, + batch_sync_rejected: false, } } @@ -667,6 +685,14 @@ mod tests { registry.is_provider_owned("pull.example.com"), "should mark pull-sync partner source" ); + assert!( + registry.rejects_batch_sync("ids.example.com"), + "should reject batch sync for a configured provider's source" + ); + assert!( + !registry.rejects_batch_sync("pull.example.com"), + "should keep batch sync for a source owned only by pull sync" + ); } #[test] @@ -727,27 +753,6 @@ mod tests { ); } - #[test] - fn batch_token_on_pull_sync_partner_is_flagged() { - let mut pull = make_partner("pull.example.com", &valid_api_token("pull-partner")); - - assert!( - !batch_sync_rejected(&pull), - "should not flag a batch-only partner" - ); - pull.pull_sync_enabled = true; - pull.pull_sync_url = Some("https://sync.example.com/pull".to_owned()); - assert!( - batch_sync_rejected(&pull), - "should flag a pull-sync partner with a batch token" - ); - pull.api_token = None; - assert!( - !batch_sync_rejected(&pull), - "should not flag a pull-sync partner without a batch token" - ); - } - #[test] fn from_ec_with_kinds_rejects_an_unregistered_kind() { let mut ec = crate::test_support::tests::create_test_settings().ec; diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index f01031f4d5..c9caa6a6e6 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -932,7 +932,7 @@ rejects, also fails startup. Lookups always verify the provider's TLS certificate, whatever `proxy.certificate_check` says. A provider owns its `sources`: its results replace whatever the row holds for -them, and batch sync rejects them and pull sync skips them. Browser-resolved IDs +them, and batch sync rejects them. Browser-resolved IDs (`ts-eids`, `sharedId`) are never stored for an owned source, so a source a provider cleared on an opt-out is not refilled from the browser. diff --git a/docs/guide/ec-setup-guide.md b/docs/guide/ec-setup-guide.md index 16207049c9..1410e3deb3 100644 --- a/docs/guide/ec-setup-guide.md +++ b/docs/guide/ec-setup-guide.md @@ -43,10 +43,8 @@ do not call either API may omit it. Provision high-entropy values under [Configuration](/guide/configuration#secret-store-migration). A partner with pull sync enabled (`pull_sync_enabled` and `pull_sync_url`) -owns its source through the pull-sync provider, so batch sync rejects every -mapping for it with `provider_owned`. Trusted Server logs a warning at config -load when such a partner also sets `api_token`; use either pull sync or batch -sync for a partner, not both. See +owns its source through the pull-sync provider, so browser EID ingestion skips +it. Batch sync keeps accepting its mappings, so a partner may use both. See [Pull Sync on Navigations](/guide/edge-cookies#pull-sync-on-navigations). Required behavior assumptions: diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index b4f2fad29c..f550ec7929 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -265,7 +265,7 @@ Server-resolved EIDs and current-request Prebid EIDs are deduplicated by `source ### Pull Sync on Navigations -Pull sync runs as an identity provider during top-level page navigations, within the shared identity lookup budget, and its result lands in the navigation's single KV write. A pull-sync partner's source is provider-owned: browser EID ingestion stores a fallback UID for it only when the row holds no live UID for that source, and batch sync rejects its mappings with `provider_owned`. Trusted Server logs a warning at config load when a pull-sync partner also has a batch `api_token`, since that token's mappings are all rejected. A source that already holds any live UID, including an ownerless fallback, is not pulled again; a source with no UID is pulled at most `pull_sync_rate_limit` times per hour per EC. +Pull sync runs as an identity provider on top-level page navigations, after the response has been sent, as it did before it became a provider. It uses the after-response budget (`identity_after_response_budget_ms`), so a slow partner never delays the page, and its result reaches the row in the follow-up write and is served from the next page load. Like other after-response lookups, it never creates a row. A pull-sync partner's source is provider-owned, so browser EID ingestion skips it. Batch sync still accepts the partner's mappings, as before. A source that already holds any live UID, including a batch-synced one, is not pulled again; a source with no UID is pulled at most `pull_sync_rate_limit` times per hour per EC. The former `ts-ec-pull-complete` completeness marker is retired. A response to a request that still carries it expires the leftover cookie. From c8f08f8c929dcf5ba912b55bba929a6f1ac2f48e Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 16:32:27 +0530 Subject: [PATCH 06/20] Set the mirror cookie only when its value changes The mirror was re-set on every consented navigation with a provider UID, and any Set-Cookie makes the response private and uncacheable, so returning users lost all caching of publisher HTML. The request's mirror value is now kept and the cookie is set only when the new value differs. An unchanged mirror is not re-extended; once it lapses the next navigation sets it again. Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/finalize.rs | 53 +++++++++++++++---- crates/trusted-server-core/src/ec/mod.rs | 37 +++++++------ 2 files changed, 65 insertions(+), 25 deletions(-) diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 86d77fe927..4ffe3fd549 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -84,9 +84,12 @@ pub fn ec_finalize_response( /// write landed. Without EC and EID consent it is expired on any response /// whose request carried it. /// -/// The cookie is only expired when the request carried it, so responses to -/// browsers without a mirror stay free of `Set-Cookie`. When the row could -/// not be read authoritatively (KV failure), the mirror is left untouched. +/// The cookie is only expired when the request carried it, and only set when +/// its value differs from the one the request carried, so responses that +/// leave the mirror as it was stay free of `Set-Cookie` and keep their +/// cacheability. An unchanged mirror is therefore not re-extended; once it +/// lapses, the next navigation sets it again. When the row could not be read +/// authoritatively (KV failure), the mirror is left untouched. fn apply_mirror( ec_context: &EcContext, registry: &PartnerRegistry, @@ -110,6 +113,7 @@ fn apply_mirror( return; }; match decision { + MirrorCookie::Set(value) if ec_context.mirror_cookie() == Some(value.as_str()) => {} MirrorCookie::Set(value) => append_cookie( response, &format!( @@ -2007,7 +2011,7 @@ mod tests { fn run_mirror_navigation( lookups: LookupResults, - mirror_present: bool, + mirror: Option<&str>, kv_read_failed: bool, ) -> Response { let settings = create_test_settings(); @@ -2030,7 +2034,7 @@ mod tests { graph.load_snapshot(&ec_id) }; let mut ec_context = returning_user_context(&ec_id, snapshot, true); - ec_context.set_mirror_cookie_present_for_test(mirror_present); + ec_context.set_mirror_cookie_for_test(mirror); ec_context.set_lookup_results(lookups); let mut registry = PartnerRegistry::from_config(&[make_partner("ids.example.com")]) .expect("should build registry"); @@ -2051,7 +2055,7 @@ mod tests { #[test] fn navigation_sets_mirror_from_committed_row() { - let response = run_mirror_navigation(provider_found("ids.example.com"), false, false); + let response = run_mirror_navigation(provider_found("ids.example.com"), None, false); let cookies = mirror_cookies(&response); assert_eq!(cookies.len(), 1, "should set one mirror cookie"); @@ -2065,9 +2069,37 @@ mod tests { ); } + #[test] + fn navigation_with_an_unchanged_mirror_emits_no_cookie() { + // Arrange + let first = run_mirror_navigation(provider_found("ids.example.com"), None, false); + let set = mirror_cookies(&first); + let value = set[0] + .strip_prefix("__Host-ts-eids-mirror=") + .and_then(|rest| rest.split(';').next()) + .expect("should set a mirror value"); + + // Act + let unchanged = + run_mirror_navigation(provider_found("ids.example.com"), Some(value), false); + let changed = + run_mirror_navigation(provider_found("ids.example.com"), Some("stale"), false); + + // Assert + assert!( + mirror_cookies(&unchanged).is_empty(), + "should not re-set a mirror whose value is unchanged" + ); + assert_eq!( + mirror_cookies(&changed).len(), + 1, + "should set the mirror when its value changed" + ); + } + #[test] fn navigation_without_provider_uids_expires_mirror() { - let response = run_mirror_navigation(LookupResults::default(), true, false); + let response = run_mirror_navigation(LookupResults::default(), Some("stale"), false); let cookies = mirror_cookies(&response); assert_eq!(cookies.len(), 1, "should emit one mirror cookie"); @@ -2079,7 +2111,7 @@ mod tests { #[test] fn navigation_without_mirror_cookie_emits_no_expiry() { - let response = run_mirror_navigation(LookupResults::default(), false, false); + let response = run_mirror_navigation(LookupResults::default(), None, false); assert!( mirror_cookies(&response).is_empty(), @@ -2089,7 +2121,8 @@ mod tests { #[test] fn kv_read_failure_leaves_mirror_untouched() { - let response = run_mirror_navigation(provider_found("ids.example.com"), true, true); + let response = + run_mirror_navigation(provider_found("ids.example.com"), Some("stale"), true); assert!( mirror_cookies(&response).is_empty(), @@ -2135,7 +2168,7 @@ mod tests { ..Default::default() }; let mut ec_context = make_context_with_consent(None, None, false, false, consent); - ec_context.set_mirror_cookie_present_for_test(true); + ec_context.set_mirror_cookie_for_test(Some("stale")); let mut response = empty_response(); ec_finalize_response( diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index fa7fe3f85b..3b70893e01 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -193,8 +193,8 @@ struct RequestEc { cookie_ec: Option, /// Whether the retired pull-sync completeness marker is present. pull_marker_present: bool, - /// Whether the mirror cookie is present. - mirror_cookie_present: bool, + /// The mirror cookie's value, when present. + mirror_cookie: Option, /// The parsed cookie jar (retained for consent pipeline input). jar: Option, } @@ -214,14 +214,15 @@ fn parse_ec_from_request(req: &Request) -> Result>, /// Whether the request carried the retired pull-sync marker cookie. pull_marker_present: bool, - /// Whether the request carried the mirror cookie. - mirror_cookie_present: bool, + /// The mirror cookie's value, when the request carried it. + mirror_cookie: Option, /// Allowed returning-user EID persistence source, assigned only after request filters pass. eid_sync_source: Option, /// Navigation permit issued by the router for page-load navigations. @@ -409,7 +410,7 @@ impl EcContext { recovery_eligible: false, client_eids: None, pull_marker_present: parsed.pull_marker_present, - mirror_cookie_present: parsed.mirror_cookie_present, + mirror_cookie: parsed.mirror_cookie, eid_sync_source: None, navigation_permit: PermitSlot::default(), after_response_permit: AfterResponsePermitSlot::default(), @@ -608,13 +609,19 @@ impl EcContext { /// Whether the request carried the mirror cookie. #[must_use] pub(crate) fn mirror_cookie_present(&self) -> bool { - self.mirror_cookie_present + self.mirror_cookie.is_some() + } + + /// The mirror cookie's value, when the request carried it. + #[must_use] + pub(crate) fn mirror_cookie(&self) -> Option<&str> { + self.mirror_cookie.as_deref() } - /// Marks the mirror cookie as present, for tests. + /// Sets the mirror cookie the request carried, for tests. #[cfg(test)] - pub(crate) fn set_mirror_cookie_present_for_test(&mut self, present: bool) { - self.mirror_cookie_present = present; + pub(crate) fn set_mirror_cookie_for_test(&mut self, value: Option<&str>) { + self.mirror_cookie = value.map(str::to_owned); } /// Marks the retired pull-sync marker cookie as present, for tests. @@ -726,7 +733,7 @@ impl EcContext { recovery_eligible: false, client_eids: None, pull_marker_present: false, - mirror_cookie_present: false, + mirror_cookie: None, eid_sync_source: None, navigation_permit: PermitSlot::default(), after_response_permit: AfterResponsePermitSlot::default(), @@ -756,7 +763,7 @@ impl EcContext { recovery_eligible: false, client_eids: None, pull_marker_present: false, - mirror_cookie_present: false, + mirror_cookie: None, eid_sync_source: None, navigation_permit: PermitSlot::default(), after_response_permit: AfterResponsePermitSlot::default(), @@ -789,7 +796,7 @@ impl EcContext { recovery_eligible: false, client_eids: None, pull_marker_present: false, - mirror_cookie_present: false, + mirror_cookie: None, eid_sync_source: None, navigation_permit: PermitSlot::default(), after_response_permit: AfterResponsePermitSlot::default(), From 024e147ace0e763682e2f45b35ac392c6d41460b Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 17:07:54 +0530 Subject: [PATCH 07/20] Skip identity lookups when the rate limiter cannot count Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 45 +++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index be66a34eab..b791281133 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -376,7 +376,7 @@ impl ProviderCoordinator

{ } /// Whether the provider's hourly limit is reached. Limiter errors fail - /// open, matching pull sync. + /// closed, as pull sync always has: an unknown count skips the lookup. /// /// The bucket is the EC hash (derived from the client IP), not the full /// EC ID, so ECs behind one IP share it and a client that clears cookies @@ -390,10 +390,10 @@ impl ProviderCoordinator

{ Ok(exceeded) => exceeded, Err(report) => { log::warn!( - "Identity lookup: provider={} rate limiter failed, allowing lookup: {report:?}", + "Identity lookup: provider={} rate limiter failed, skipping lookup: {report:?}", registered.id ); - false + true } } } @@ -1895,6 +1895,45 @@ mod tests { assert!(transport.sent.borrow().is_empty(), "should send nothing"); } + struct FailingLimiter; + + impl RateLimiter for FailingLimiter { + fn exceeded( + &self, + _key: &str, + _hourly_limit: u32, + ) -> Result> { + Err(Report::new(TrustedServerError::KvStore { + store_name: "example-counter".to_owned(), + message: "unavailable".to_owned(), + })) + } + } + + #[test] + fn limiter_failure_skips_the_lookup() { + let coordinator = coordinator_with(RegisteredProvider::new( + provider_config("rate_limit_per_hour = 1"), + echo_provider(SourceInput::Anonymous), + )); + let transport = CannedTransport::new(ok_reply); + + let results = run_with( + &coordinator, + &transport, + &missing(), + &granting(), + &FailingLimiter, + ); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::RateLimited), + "should skip when the limiter cannot count" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + #[test] fn built_url_outside_allowlist_is_rejected() { let coordinator = coordinator(""); From 4c5beacd9ecc190cb88b51e0add114f534d8863e Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 17:09:03 +0530 Subject: [PATCH 08/20] Reject the pull-sync kind in identity provider config Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/config.rs | 31 +++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index caf324e02a..660b0ea23a 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -15,7 +15,7 @@ use url::{Host, Url}; use validator::{Validate, ValidationError, ValidationErrors}; use crate::ec::provider::validate_enabled_configs; -use crate::ec::registry::PartnerRegistry; +use crate::ec::registry::{PULL_SYNC_KIND, PartnerRegistry}; use crate::error::TrustedServerError; use crate::integrations::{ adserver_mock::AdServerMockConfig, @@ -448,6 +448,20 @@ fn validate_identity_providers(ec: &Ec) -> Result<(), Report if ec.identity_providers.is_empty() { return Ok(()); } + // Pull-sync providers come only from partner config, which carries the + // pull token and rate limit; configuring the kind directly bypasses both. + if let Some(config) = ec + .identity_providers + .iter() + .find(|config| config.kind() == PULL_SYNC_KIND) + { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "ec.identity_providers: provider `{}` uses kind `{PULL_SYNC_KIND}`; configure pull sync on its [[ec.partners]] entry instead", + config.id + ), + })); + } let partners = PartnerRegistry::from_ec(ec)?; validate_enabled_configs(&ec.identity_providers, &partners).map_err(|report| { let detail = report @@ -1708,6 +1722,21 @@ gam_network_id = "99999" ); } + #[test] + fn runtime_validation_rejects_a_directly_configured_pull_sync_provider() { + let settings = settings_with_identity_provider(serde_json::json!({ + "kind": PULL_SYNC_KIND, + })); + + let err = validate_settings_for_runtime(&settings) + .expect_err("should reject the pull-sync kind in identity providers"); + + assert!( + err.to_string().contains("[[ec.partners]]"), + "should point to partner config: {err:?}" + ); + } + #[test] fn runtime_validation_rejects_provider_source_without_partner() { let settings = settings_with_identity_provider(serde_json::json!({ From 864f95d9487f78b4135923f698f42ddf03b915c7 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 17:09:40 +0530 Subject: [PATCH 09/20] Refuse to key the template cache on the EID mirror cookie Signed-off-by: dhruv8sh --- .../src/cookies/template_cache_policy.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-core/src/cookies/template_cache_policy.rs b/crates/trusted-server-core/src/cookies/template_cache_policy.rs index 61c3b9139c..3997604534 100644 --- a/crates/trusted-server-core/src/cookies/template_cache_policy.rs +++ b/crates/trusted-server-core/src/cookies/template_cache_policy.rs @@ -2,7 +2,7 @@ use std::collections::{HashMap, HashSet}; -use crate::constants::{COOKIE_SHAREDID, COOKIE_TS_EC, COOKIE_TS_EIDS}; +use crate::constants::{COOKIE_SHAREDID, COOKIE_TS_EC, COOKIE_TS_EIDS, COOKIE_TS_EIDS_MIRROR}; use crate::platform::TemplateCookieValue; /// Whether the request can share a template and its explicit cookie dimensions. @@ -32,7 +32,13 @@ pub(crate) fn validate_cookie_names( return Err(format!("{field} contains invalid cookie name `{name}`")); } if field == "template_cache_key_cookies" - && [COOKIE_TS_EC, COOKIE_TS_EIDS, COOKIE_SHAREDID].contains(&name.as_str()) + && [ + COOKIE_TS_EC, + COOKIE_TS_EIDS, + COOKIE_TS_EIDS_MIRROR, + COOKIE_SHAREDID, + ] + .contains(&name.as_str()) { return Err(format!( "{field} must not key on Trusted Server identity cookie `{name}`" @@ -232,7 +238,12 @@ mod tests { #[test] fn template_cookie_policy_rejects_identity_keys_but_allows_bypass() { - for name in [COOKIE_TS_EC, COOKIE_TS_EIDS, COOKIE_SHAREDID] { + for name in [ + COOKIE_TS_EC, + COOKIE_TS_EIDS, + COOKIE_TS_EIDS_MIRROR, + COOKIE_SHAREDID, + ] { let error = validate_cookie_names(&names(&[name]), &[]) .expect_err("should reject identity cookie keys"); assert!( From 902b08d57af0884aa2df471f0a5fe08b48332319 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 17:11:02 +0530 Subject: [PATCH 10/20] Rotate pull-sync partners hourly so every partner gets a turn under the cap Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index b791281133..fcd08e1b10 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -239,6 +239,22 @@ impl ProviderCoordinator

{ }) } + /// The order providers are dispatched in: configured providers first, + /// then pull-sync partners rotated by the hour, so that under + /// `pull_sync_concurrency` every partner gets a turn, as pull sync did + /// before it became a provider. + fn dispatch_order(&self, now: UnixSeconds) -> Vec { + let (mut order, mut pull_sync): (Vec, Vec) = + (0..self.providers.len()).partition(|&index| !self.providers[index].policy.pull_sync); + if !pull_sync.is_empty() { + let hour = usize::try_from(now.get() / 3600).unwrap_or(0); + let offset = hour % pull_sync.len(); + pull_sync.rotate_left(offset); + } + order.append(&mut pull_sync); + order + } + /// Caps the pull-sync lookups dispatched per navigation /// (`ec.pull_sync_concurrency`). Due pull-sync partners past the cap are /// skipped as rate limited and retried on a later navigation. @@ -557,7 +573,8 @@ impl IdentityCoordinator for ProviderCoordinator

{ let deadline = Deadline::after(self.budget); pending.deadline = Some(deadline); let mut pull_sync_dispatched = 0_usize; - for (index, registered) in self.providers.iter().enumerate() { + for index in self.dispatch_order(lookup.now) { + let registered = &self.providers[index]; let provider = ProviderIndex::new(index); let skip = |reason| Tagged { provider, @@ -1527,6 +1544,23 @@ mod tests { ); } + #[test] + fn pull_sync_rotates_the_partner_past_the_cap_by_the_hour() { + let coordinator = pull_sync_pair(NonZeroUsize::new(1)); + + let first_hour = coordinator.dispatch_order(UnixSeconds::new(0)); + let next_hour = coordinator.dispatch_order(UnixSeconds::new(3600)); + + assert_ne!( + first_hour.first(), + next_hour.first(), + "should give the other partner the first slot the next hour" + ); + let mut sorted = next_hour.clone(); + sorted.sort_unstable(); + assert_eq!(sorted, vec![0, 1], "should still dispatch every provider"); + } + #[test] fn pull_sync_holds_ownerless_uids_without_repulling() { let coordinator = pull_sync_pair(None); From e12d56ed798697c76ea3b10c574c9ebdd16b3df4 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 17:11:02 +0530 Subject: [PATCH 11/20] Document pull-sync rate limit rounding, the per-navigation cap and mirror exposure Signed-off-by: dhruv8sh --- docs/guide/edge-cookies.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index f550ec7929..17246fa13c 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -265,7 +265,7 @@ Server-resolved EIDs and current-request Prebid EIDs are deduplicated by `source ### Pull Sync on Navigations -Pull sync runs as an identity provider on top-level page navigations, after the response has been sent, as it did before it became a provider. It uses the after-response budget (`identity_after_response_budget_ms`), so a slow partner never delays the page, and its result reaches the row in the follow-up write and is served from the next page load. Like other after-response lookups, it never creates a row. A pull-sync partner's source is provider-owned, so browser EID ingestion skips it. Batch sync still accepts the partner's mappings, as before. A source that already holds any live UID, including a batch-synced one, is not pulled again; a source with no UID is pulled at most `pull_sync_rate_limit` times per hour per EC. +Pull sync runs as an identity provider on top-level page navigations, after the response has been sent, as it did before it became a provider. It uses the after-response budget (`identity_after_response_budget_ms`), so a slow partner never delays the page, and its result reaches the row in the follow-up write and is served from the next page load. Like other after-response lookups, it never creates a row. A pull-sync partner's source is provider-owned, so browser EID ingestion skips it. Batch sync still accepts the partner's mappings, as before. A source that already holds any live UID, including a batch-synced one, is not pulled again; a source with no UID is pulled at most `pull_sync_rate_limit` times per hour per EC. On Fastly that limit is enforced as a per-minute budget of at least one, so values below 60 behave like 60 per hour. At most `pull_sync_concurrency` partners are pulled per navigation (0 means no limit), unlike before, when every eligible partner was pulled in batches of that size. The partner order rotates every hour, so partners past the cap still get turns. The former `ts-ec-pull-complete` completeness marker is retired. A response to a request that still carries it expires the leftover cookie. @@ -279,6 +279,7 @@ Trusted Server mirrors provider-written UIDs for `bidstream_enabled` partners in - **Attributes**: `Path=/; Secure; SameSite=Lax; Max-Age=`, host-only (no `Domain`) and not `HttpOnly` - **Contents**: unpadded base64url JSON `{"v":1,"eids":[{"source":"ids.example.com","uids":[{"id":"...","atype":1,"exp":1767225600}]}]}`, partners in `[[ec.partners]]` order, at most 3584 bytes; UIDs that do not fit are left out - **Contents source**: live UIDs written by an identity provider. Ownerless UIDs, from browser ingest, batch sync or legacy pull sync, are never mirrored. +- **Pull-sync UIDs**: pull sync now writes provider-owned UIDs, so a `bidstream_enabled` pull-sync partner's UIDs are mirrored. Before, they stayed in KV only; now any script on the page can read them, and the cookie is sent on every same-origin request. Set `bidstream_enabled = false` on a partner whose UIDs must not reach the page. - **Updates** (best effort): page navigations rebuild it from the row after that navigation's write. It is expired only when the request carried it, either because no live provider UID qualifies or because EC and EID consent is missing (on any response). Responses to browsers without a mirror get no mirror `Set-Cookie`. When the identity row cannot be read (KV failure) the mirror is left untouched, and a mirror that cannot be built is skipped without failing the response. Readers must drop UIDs whose `exp` (Unix seconds) has passed and strip `exp` before using a UID. From a7e67a1569c66c8cb406d1608c8c19b27a83f123 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 17:12:12 +0530 Subject: [PATCH 12/20] Test mirroring of pull-sync results, non-navigations and withdrawn rows Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/finalize.rs | 47 ++++++++++++++++++- crates/trusted-server-core/src/ec/mirror.rs | 18 +++++++ 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 4ffe3fd549..ab8646b09b 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -1984,11 +1984,15 @@ mod tests { } fn provider_found(source: &str) -> LookupResults { + provider_found_by(source, "example-provider") + } + + fn provider_found_by(source: &str, owner: &str) -> LookupResults { let uid = KvUid::new( UidValue::try_from("provider-uid").expect("should build uid"), AgentType::try_from(1).expect("should build atype"), None, - Some(ProviderId::new("example-provider")), + Some(ProviderId::new(owner)), Expiry::Never, ); // Made for the EC ID `run_mirror_navigation` seeds. @@ -2013,6 +2017,15 @@ mod tests { lookups: LookupResults, mirror: Option<&str>, kv_read_failed: bool, + ) -> Response { + run_mirror_request(lookups, mirror, kv_read_failed, true) + } + + fn run_mirror_request( + lookups: LookupResults, + mirror: Option<&str>, + kv_read_failed: bool, + navigation: bool, ) -> Response { let settings = create_test_settings(); let ec_id = sample_ec_id("mirr01"); @@ -2033,7 +2046,7 @@ mod tests { } else { graph.load_snapshot(&ec_id) }; - let mut ec_context = returning_user_context(&ec_id, snapshot, true); + let mut ec_context = returning_user_context(&ec_id, snapshot, navigation); ec_context.set_mirror_cookie_for_test(mirror); ec_context.set_lookup_results(lookups); let mut registry = PartnerRegistry::from_config(&[make_partner("ids.example.com")]) @@ -2069,6 +2082,36 @@ mod tests { ); } + #[test] + fn pull_sync_result_is_mirrored_with_its_owner() { + let response = run_mirror_navigation( + provider_found_by("ids.example.com", "pull-sync:ids.example.com"), + None, + false, + ); + + assert_eq!( + mirror_cookies(&response).len(), + 1, + "should mirror a provider-owned pull-sync UID" + ); + } + + #[test] + fn consented_non_navigation_leaves_the_mirror_untouched() { + let response = run_mirror_request( + provider_found("ids.example.com"), + Some("stale"), + false, + false, + ); + + assert!( + mirror_cookies(&response).is_empty(), + "should neither set nor expire the mirror outside a navigation" + ); + } + #[test] fn navigation_with_an_unchanged_mirror_emits_no_cookie() { // Arrange diff --git a/crates/trusted-server-core/src/ec/mirror.rs b/crates/trusted-server-core/src/ec/mirror.rs index f26b2f41cb..e5a026a700 100644 --- a/crates/trusted-server-core/src/ec/mirror.rs +++ b/crates/trusted-server-core/src/ec/mirror.rs @@ -359,6 +359,24 @@ mod tests { ); } + #[test] + fn tombstoned_row_expires_the_mirror_even_with_consent_on_the_request() { + let mut entry = row(vec![uid("live", Expiry::Never)]); + entry.consent.ok = false; + + let decision = build_mirror( + Some(&entry), + &granting(), + &partners(true), + UnixSeconds::new(1000), + ); + + assert!( + matches!(decision, MirrorCookie::Expire), + "should expire the mirror for a withdrawn row" + ); + } + #[test] fn mirrors_several_eids_as_valid_json() { let mut entry = row(vec![uid("one", Expiry::Never), uid("two", Expiry::Never)]); From a6f397fcbef6e6a4e61365073138b176c41e79a3 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Tue, 10 Nov 2026 19:03:42 +0530 Subject: [PATCH 13/20] Test the pull-sync UID cap at the provider write limit Pull sync now runs as a provider, so its UIDs are capped at MAX_STORED_UID_LENGTH like every provider write after the base branch split the ingest and stored limits. Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/coordinator/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index fcd08e1b10..28307b5e59 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -1605,7 +1605,9 @@ mod tests { Ok(TransportResponse { status: http::StatusCode::OK, headers: http::HeaderMap::new(), - body: bytes::Bytes::from("x".repeat(crate::ec::kv_types::MAX_UID_LENGTH + 1)), + body: bytes::Bytes::from( + "x".repeat(crate::ec::kv_types::MAX_STORED_UID_LENGTH + 1), + ), }) }); From 7360a0e6d1fc5166303cb2c8ec3a363b83861249 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Tue, 10 Nov 2026 19:15:31 +0530 Subject: [PATCH 14/20] Address review of pull sync and the EID mirror Mirror: add a per-partner mirror_enabled opt-in, off by default, so a partner's UIDs can stay in the bidstream without reaching page scripts; leave out UIDs whose owning provider's TCF vendor consent is denied, as live_eids does for the bidstream. Pull sync: restore browser ts-eids and sharedId ingest for sources owned only by pull sync (the batch-sync split now covers browser ingest too, as ingest_rejected); gate after-response lookups on the real-browser check in prepare_after_response_lookups; skip the after-response row read when the request snapshot has nothing due (ProviderCoordinator::any_due) and the follow-up re-read when no result has anything to persist; log the per-navigation cap as skipped_concurrency_cap. Config: reject at startup a ts_pull_token that cannot form an Authorization header; define EcPartner::runs_pull_sync once and use it for ownership, the double-claim check and provider building; remove the unused pull_enabled_partners; cover the disabled-provider bypass of the double-claim check. Docs: describe where pull sync runs now, the mirror opt-in and vendor consent. Signed-off-by: dhruv8sh --- .../trusted-server-adapter-fastly/src/app.rs | 63 ++++++++- .../trusted-server-adapter-fastly/src/main.rs | 10 +- .../src/auction/endpoints.rs | 1 + crates/trusted-server-core/src/config.rs | 56 +++++++- crates/trusted-server-core/src/ec/admin.rs | 1 + .../src/ec/after_response.rs | 34 +++++ crates/trusted-server-core/src/ec/auth.rs | 1 + .../trusted-server-core/src/ec/batch_sync.rs | 5 +- .../src/ec/coordinator/mod.rs | 47 ++++++- .../src/ec/coordinator/results.rs | 4 + crates/trusted-server-core/src/ec/eids.rs | 1 + crates/trusted-server-core/src/ec/finalize.rs | 10 +- crates/trusted-server-core/src/ec/identify.rs | 1 + crates/trusted-server-core/src/ec/mirror.rs | 91 +++++++++++-- .../trusted-server-core/src/ec/prebid_eids.rs | 1 + crates/trusted-server-core/src/ec/registry.rs | 120 ++++++++++-------- .../trusted-server-core/src/ec/write_plan.rs | 33 ++++- crates/trusted-server-core/src/settings.rs | 14 ++ .../src/lib.rs | 10 +- docs/guide/configuration.md | 1 + docs/guide/ec-setup-guide.md | 5 +- docs/guide/edge-cookies.md | 7 +- trusted-server.example.toml | 3 + 23 files changed, 438 insertions(+), 81 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index cadd3d4665..963132be41 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -461,14 +461,19 @@ fn joined_cookie_header(req: &Request) -> String { /// permitted publisher navigation. /// /// Returns `None` when no after-response provider (including a pull-sync -/// partner) is enabled, the request has no EC, or the identity graph is -/// unavailable. +/// partner) is enabled, the client failed the browser check, the request has +/// no EC, or the identity graph is unavailable. fn prepare_after_response_lookups( state: &AppState, req: &Request, ec: &EcRequestState, ) -> Option { state.after_response_identity.as_ref()?; + // Bot gate: pull sync and other after-response lookups disclose the EC ID + // to partners, so they run only for recognized browsers. + if !ec.is_real_browser { + return None; + } ec.kv_graph.as_ref()?; let ec_id = ec.ec_context.ec_value()?.to_owned(); let signals = ProviderRequestSignals::read(req, ec); @@ -3478,6 +3483,60 @@ mod tests { ); } + #[test] + fn after_response_lookups_run_only_for_real_browsers() { + let mut settings = test_settings(); + settings.ec.ec_store = Some("ec_identity_store".to_owned()); + settings.ec.partners = vec![ + serde_json::from_value(serde_json::json!({ + "name": "Pull partner", + "source_domain": "partner.example.com", + "pull_sync_enabled": true, + "pull_sync_url": "https://sync.example.com/pull", + "pull_sync_allowed_domains": ["sync.example.com"], + "ts_pull_token": "example-pull-token-32-bytes-minimum", + })) + .expect("should build pull-sync partner"), + ]; + let state = build_state_from_settings(settings).expect("should build app state"); + let browser_request = || { + let mut req = request_builder() + .method(Method::GET) + .uri("https://www.example.com/article") + .header(header::COOKIE, format!("ts-ec={}.test01", "a".repeat(64))) + .body(Body::empty()) + .expect("should build request"); + req.extensions_mut().insert(DeviceSignals::derive( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36", + Some("t13d1516h2_8daaf6152771_b186095e22b6"), + Some("1:65536;2:0;4:6291456;6:262144"), + )); + req + }; + let context = RequestContext::new(browser_request(), PathParams::default()); + let services = build_per_request_services(&state, &context); + let req = browser_request(); + let mut ec = super::build_ec_request_state(&state.settings, &services, &req); + assert!( + ec.is_real_browser, + "should classify the request as a browser" + ); + + let browser = super::prepare_after_response_lookups(&state, &req, &ec); + ec.is_real_browser = false; + let bot = super::prepare_after_response_lookups(&state, &req, &ec); + + assert!( + browser.is_some(), + "should prepare pull sync for a real browser" + ); + assert!( + bot.is_none(), + "should never prepare pull sync for a client that fails the browser check" + ); + } + #[test] fn missing_device_signals_extension_classifies_as_non_browser() { // Without the entry-point-captured signals, the reconstructed request diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index acb0b411ea..582f11a6ad 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -739,6 +739,15 @@ pub(crate) fn run_after_response_lookups( return; }; let settings = &*state.settings; + // The request snapshot holds the row as finalize left it. A source held + // there is still held in the live row unless consent was withdrawn, which + // blocks the lookup anyway, so the live read is skipped when nothing is due. + let now = current_timestamp(); + let request_entry = ec_state.ec_context.kv_snapshot().entry_for(&lookups.ec_id); + if !coordinator.any_due(request_entry, UnixSeconds::new(now)) { + log::debug!("After-response lookups skipped: nothing due"); + return; + } let Some(graph) = maybe_identity_graph(settings) else { return; }; @@ -753,7 +762,6 @@ pub(crate) fn run_after_response_lookups( } let transport = PlatformLookupTransport::new(&ec_state.services); let limiter = FastlyRateLimiter::new(RATE_COUNTER_NAME); - let now = current_timestamp(); let pending = futures::executor::block_on(coordinator.dispatch( &transport, &LookupContext { diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 310b41bf98..4d4630dcec 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -955,6 +955,7 @@ mod tests { source_domain: source_domain.to_owned(), openrtb_atype: crate::settings::EcPartner::default_openrtb_atype(), bidstream_enabled: true, + mirror_enabled: false, api_token: Some(crate::redacted::Redacted::new(format!( "token-{source_domain}-32-bytes-minimum-value" ))), diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 660b0ea23a..39943d5516 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -478,9 +478,17 @@ fn validate_identity_providers(ec: &Ec) -> Result<(), Report // fail the coordinator build on every request. for config in ec.identity_providers.iter().filter(|config| config.enabled) { if let Some(source) = config.sources.iter().find(|source| { - partners + let claimed = partners .find_by_source_domain(source) - .is_some_and(|partner| partner.pull_sync_enabled && partner.pull_sync_url.is_some()) + .map(|partner| partner.source_domain.as_str()); + claimed.is_some() + && ec.partners.iter().any(|partner| { + partner.runs_pull_sync() + && partners + .find_by_source_domain(&partner.source_domain) + .map(|config| config.source_domain.as_str()) + == claimed + }) }) { return Err(Report::new(TrustedServerError::Configuration { message: format!( @@ -1722,6 +1730,50 @@ gam_network_id = "99999" ); } + #[test] + fn runtime_validation_accepts_disabled_provider_sharing_a_pull_sync_source() { + let mut settings = settings_with_identity_provider(serde_json::json!({ "enabled": false })); + settings.ec.partners = vec![ + serde_json::from_value(serde_json::json!({ + "name": "ids", + "source_domain": "ids.example.com", + "pull_sync_enabled": true, + "pull_sync_url": "https://sync.example.com/pull", + "pull_sync_allowed_domains": ["sync.example.com"], + "ts_pull_token": "example-pull-token-32-bytes-minimum", + })) + .expect("should build partner"), + ]; + + validate_settings_for_runtime(&settings) + .expect("should accept a disabled provider sharing a pull-sync source"); + } + + #[test] + fn runtime_validation_rejects_a_pull_token_invalid_in_a_header() { + let mut settings = settings_with_identity_provider(serde_json::json!({ "enabled": false })); + settings.ec.partners = vec![ + serde_json::from_value(serde_json::json!({ + "name": "ids", + "source_domain": "ids.example.com", + "pull_sync_enabled": true, + "pull_sync_url": "https://sync.example.com/pull", + "pull_sync_allowed_domains": ["sync.example.com"], + "ts_pull_token": "example-pull-token\n32-bytes-minimum", + })) + .expect("should build partner"), + ]; + + let err = validate_settings_for_runtime(&settings) + .expect_err("should reject a pull token that cannot form a header"); + + assert!( + err.to_string().contains("pull sync config invalid") + || format!("{err:?}").contains("Authorization header"), + "should explain the invalid pull token: {err:?}" + ); + } + #[test] fn runtime_validation_rejects_a_directly_configured_pull_sync_provider() { let settings = settings_with_identity_provider(serde_json::json!({ diff --git a/crates/trusted-server-core/src/ec/admin.rs b/crates/trusted-server-core/src/ec/admin.rs index 7b8493beb2..e6f219b75e 100644 --- a/crates/trusted-server-core/src/ec/admin.rs +++ b/crates/trusted-server-core/src/ec/admin.rs @@ -735,6 +735,7 @@ mod tests { source_domain: source_domain.to_owned(), openrtb_atype: EcPartner::default_openrtb_atype(), bidstream_enabled, + mirror_enabled: false, api_token: Some(Redacted::new(format!("test-token-{source_domain:-<32}"))), batch_rate_limit: EcPartner::default_batch_rate_limit(), pull_sync_enabled: false, diff --git a/crates/trusted-server-core/src/ec/after_response.rs b/crates/trusted-server-core/src/ec/after_response.rs index 5e5e0f9636..58facd44f8 100644 --- a/crates/trusted-server-core/src/ec/after_response.rs +++ b/crates/trusted-server-core/src/ec/after_response.rs @@ -204,6 +204,40 @@ mod tests { .and_then(|entry| entry.ids.get(source).map(|record| record.uid.clone())) } + #[test] + fn results_with_nothing_to_persist_skip_the_row_read() { + let reads = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let graph = KvIdentityGraph::counting("counting-store", std::sync::Arc::clone(&reads)); + let not_due = LookupResults::new( + EC_ID, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Skipped(SkipReason::NotDue), + entries: Vec::new(), + cleared: Vec::new(), + }], + ); + + let outcome = commit_after_response_lookups( + &graph, + AfterResponseWrite::for_test(), + ®istry(), + EC_ID, + ¬_due, + ); + + assert_eq!( + outcome, + AfterResponseOutcome::Unchanged, + "should report nothing to write" + ); + assert_eq!( + reads.load(std::sync::atomic::Ordering::Relaxed), + 0, + "should not re-read the row when every provider was not due" + ); + } + #[test] fn writes_after_response_results_to_an_existing_row() { let graph = KvIdentityGraph::in_memory("after-response"); diff --git a/crates/trusted-server-core/src/ec/auth.rs b/crates/trusted-server-core/src/ec/auth.rs index b7db6f1fa8..52b616be1a 100644 --- a/crates/trusted-server-core/src/ec/auth.rs +++ b/crates/trusted-server-core/src/ec/auth.rs @@ -52,6 +52,7 @@ mod tests { source_domain: source_domain.to_owned(), openrtb_atype: EcPartner::default_openrtb_atype(), bidstream_enabled: true, + mirror_enabled: false, api_token: Some(Redacted::new(api_token.to_owned())), batch_rate_limit: EcPartner::default_batch_rate_limit(), pull_sync_enabled: false, diff --git a/crates/trusted-server-core/src/ec/batch_sync.rs b/crates/trusted-server-core/src/ec/batch_sync.rs index e9b2b319e0..19f5f45f72 100644 --- a/crates/trusted-server-core/src/ec/batch_sync.rs +++ b/crates/trusted-server-core/src/ec/batch_sync.rs @@ -172,7 +172,7 @@ fn handle_batch_sync_with_writer( let (accepted, errors) = process_mappings_for_partner( writer, &partner.source_domain, - registry.rejects_batch_sync(&partner.source_domain), + registry.rejects_ingest(&partner.source_domain), &body.mappings, ); @@ -438,6 +438,7 @@ mod tests { source_domain: source_domain.to_owned(), openrtb_atype: EcPartner::default_openrtb_atype(), bidstream_enabled: true, + mirror_enabled: false, api_token: Some(Redacted::new(api_token.to_owned())), batch_rate_limit: EcPartner::default_batch_rate_limit(), pull_sync_enabled: false, @@ -1034,7 +1035,7 @@ mod tests { fn handle_batch_sync_rejects_mappings_for_a_configured_provider_source() { let mut registry = test_registry(); assert!( - registry.reject_batch_sync("ssp.example.com"), + registry.reject_ingest("ssp.example.com"), "should mark the configured provider source" ); let limiter = MockRateLimiter { diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 28307b5e59..85f3c3c640 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -319,6 +319,17 @@ impl ProviderCoordinator

{ } /// Whether the row holds a live provider-written UID for `source`. + /// Whether any provider has a due source in `entry`, ignoring consent and + /// rate limits, so a caller can skip a row read that cannot lead to a + /// lookup. `None` (no row known) counts every source as due. + #[must_use] + pub fn any_due(&self, entry: Option<&KvEntry>, now: UnixSeconds) -> bool { + self.providers.iter().any(|registered| { + !entry.is_some_and(|entry| entry.opted_out(®istered.id, now)) + && !Self::due_sources(registered, entry, now).0.is_empty() + }) + } + fn holds_live_uid(entry: Option<&KvEntry>, source: &str, now: UnixSeconds) -> bool { entry .and_then(|entry| entry.ids.get(source)) @@ -721,7 +732,7 @@ impl IdentityCoordinator for ProviderCoordinator

{ .pull_sync_concurrency .is_some_and(|max| pull_sync_dispatched >= max.get()) { - pending.early.push(skip(SkipReason::RateLimited)); + pending.early.push(skip(SkipReason::ConcurrencyCap)); continue; } let input = LookupInput { @@ -1528,6 +1539,33 @@ mod tests { .with_pull_sync_concurrency(max) } + #[test] + fn any_due_is_false_only_when_every_source_is_held() { + let coordinator = pull_sync_pair(None); + let now = UnixSeconds::new(1000); + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry + .ids + .insert("a.example.com".to_owned(), KvPartnerId::single("held-a")); + + assert!( + coordinator.any_due(None, now), + "should count every source as due without a known row" + ); + assert!( + coordinator.any_due(Some(&entry), now), + "should report the source that is still missing" + ); + entry + .ids + .insert("b.example.com".to_owned(), KvPartnerId::single("held-b")); + assert!( + !coordinator.any_due(Some(&entry), now), + "should report nothing due once every pull-sync source is held" + ); + } + #[test] fn pull_sync_concurrency_caps_dispatched_lookups() { let coordinator = pull_sync_pair(NonZeroUsize::new(1)); @@ -1539,9 +1577,14 @@ mod tests { let statuses: Vec = results.iter().map(|lookup| lookup.status).collect(); assert_eq!( statuses[1], - LookupStatus::Skipped(SkipReason::RateLimited), + LookupStatus::Skipped(SkipReason::ConcurrencyCap), "should skip the pull-sync lookup past the cap" ); + assert_eq!( + statuses[1].metric_name(), + "skipped_concurrency_cap", + "should log the cap apart from the rate limit" + ); } #[test] diff --git a/crates/trusted-server-core/src/ec/coordinator/results.rs b/crates/trusted-server-core/src/ec/coordinator/results.rs index 9c3a90bb60..fa67dc0982 100644 --- a/crates/trusted-server-core/src/ec/coordinator/results.rs +++ b/crates/trusted-server-core/src/ec/coordinator/results.rs @@ -63,6 +63,9 @@ pub enum SkipReason { Destination, /// The provider's hourly rate limit is reached. RateLimited, + /// Pull sync already dispatched `pull_sync_concurrency` partners on this + /// navigation. + ConcurrencyCap, /// The vendor or its cookies report an opt-out; the provider's stored /// UIDs are removed. OptedOut, @@ -118,6 +121,7 @@ impl LookupStatus { Self::Skipped(SkipReason::Input) => "skipped_input", Self::Skipped(SkipReason::RateLimited) => "skipped_rate_limited", Self::Skipped(SkipReason::Region) => "skipped_region", + Self::Skipped(SkipReason::ConcurrencyCap) => "skipped_concurrency_cap", Self::Skipped(SkipReason::SnapshotUnavailable) => "skipped_snapshot_unavailable", Self::Skipped(SkipReason::Destination) => "destination_rejected", Self::Skipped(SkipReason::OptedOut) => "skipped_opted_out", diff --git a/crates/trusted-server-core/src/ec/eids.rs b/crates/trusted-server-core/src/ec/eids.rs index f1073b2442..70d56054e4 100644 --- a/crates/trusted-server-core/src/ec/eids.rs +++ b/crates/trusted-server-core/src/ec/eids.rs @@ -192,6 +192,7 @@ mod tests { source_domain: source_domain.to_owned(), openrtb_atype: EcPartner::default_openrtb_atype(), bidstream_enabled: true, + mirror_enabled: false, api_token: Some(Redacted::new(format!( "token-{source_domain}-32-bytes-minimum-value" ))), diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index ab8646b09b..9f121e1795 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -732,6 +732,7 @@ mod tests { source_domain: source_domain.to_owned(), openrtb_atype: EcPartner::default_openrtb_atype(), bidstream_enabled: true, + mirror_enabled: false, api_token: Some(Redacted::new(format!( "token-{source_domain}-32-bytes-minimum-value" ))), @@ -1616,6 +1617,10 @@ mod tests { registry.mark_provider_owned("sharedid.org", None), "should mark the provider source" ); + assert!( + registry.reject_ingest("sharedid.org"), + "should reject browser ingest for the configured provider's source" + ); let mut response = empty_response(); // Act @@ -2049,8 +2054,9 @@ mod tests { let mut ec_context = returning_user_context(&ec_id, snapshot, navigation); ec_context.set_mirror_cookie_for_test(mirror); ec_context.set_lookup_results(lookups); - let mut registry = PartnerRegistry::from_config(&[make_partner("ids.example.com")]) - .expect("should build registry"); + let mut partner = make_partner("ids.example.com"); + partner.mirror_enabled = true; + let mut registry = PartnerRegistry::from_config(&[partner]).expect("should build registry"); registry.mark_provider_owned("ids.example.com", None); let mut response = empty_response(); diff --git a/crates/trusted-server-core/src/ec/identify.rs b/crates/trusted-server-core/src/ec/identify.rs index ed36371bbd..23f6e11f2d 100644 --- a/crates/trusted-server-core/src/ec/identify.rs +++ b/crates/trusted-server-core/src/ec/identify.rs @@ -376,6 +376,7 @@ mod tests { source_domain: source_domain.to_owned(), openrtb_atype: EcPartner::default_openrtb_atype(), bidstream_enabled: true, + mirror_enabled: false, api_token: Some(Redacted::new(api_token.to_owned())), batch_rate_limit: EcPartner::default_batch_rate_limit(), pull_sync_enabled: false, diff --git a/crates/trusted-server-core/src/ec/mirror.rs b/crates/trusted-server-core/src/ec/mirror.rs index e5a026a700..3cf82746eb 100644 --- a/crates/trusted-server-core/src/ec/mirror.rs +++ b/crates/trusted-server-core/src/ec/mirror.rs @@ -4,7 +4,7 @@ use base64::Engine as _; use serde_json::{Map, Value}; use crate::consent::ConsentContext; -use crate::consent::allows_eid_persistence; +use crate::consent::{allows_eid_persistence, tcf_vendor_consent_denied}; use crate::ec::consent::ec_consent_granted; use crate::ec::eid_types::{Expiry, UnixSeconds}; use crate::ec::kv_types::{KvEid, KvEntry, KvUid}; @@ -36,7 +36,9 @@ impl MirrorValue { /// Builds the mirror from the row as it stands after this navigation. /// -/// Holds live, provider-written UIDs of `bidstream_enabled` partners, in +/// Holds live, provider-written UIDs of partners with both +/// `bidstream_enabled` and `mirror_enabled` whose owning provider's TCF vendor +/// consent is not denied, in /// partner config order, as unpadded base64url JSON /// `{"v":1,"eids":[...]}`. UIDs that would push the value past /// [`MAX_MIRROR_BYTES`] are skipped, as are UIDs that fail to serialize. @@ -59,10 +61,16 @@ pub fn build_mirror( return MirrorCookie::Expire; }; let mut mirror = MirrorBuilder::default(); - for partner in partners - .in_config_order() - .filter(|partner| partner.bidstream_enabled) - { + // A partner opts in with `mirror_enabled`, and a UID never reaches the + // page while its owning provider's TCF vendor consent is denied, as it + // never reaches the bidstream (`KvEntry::live_eids`). + for partner in partners.in_config_order().filter(|partner| { + partner.bidstream_enabled + && partner.mirror_enabled + && !partner + .owner_tcf_vendor_id + .is_some_and(|vendor| tcf_vendor_consent_denied(consent, vendor)) + }) { let Some(record) = row.ids.get(&partner.source_domain) else { continue; }; @@ -230,11 +238,18 @@ mod tests { use super::*; use crate::ec::eid_types::{AgentType, EidFields, ProviderId, UidValue}; use crate::ec::kv_types::KvPartnerId; + use crate::test_support::tests::consent_granting_vendors; use serde_json::json; + use std::num::NonZeroU16; fn partners(bidstream: bool) -> PartnerRegistry { + partners_with(bidstream, true) + } + + fn partners_with(bidstream: bool, mirror: bool) -> PartnerRegistry { let partner = serde_json::from_value(json!({ - "name": "ids", "source_domain": "ids.example.com", "bidstream_enabled": bidstream + "name": "ids", "source_domain": "ids.example.com", + "bidstream_enabled": bidstream, "mirror_enabled": mirror })) .expect("should build partner"); PartnerRegistry::from_config(&[partner]).expect("should build registry") @@ -332,6 +347,66 @@ mod tests { ); } + #[test] + fn mirror_is_off_unless_the_partner_opts_in() { + let entry = row(vec![uid("live", Expiry::Never)]); + + let mirror = build_mirror( + Some(&entry), + &granting(), + &partners_with(true, false), + UnixSeconds::new(1000), + ); + + assert_eq!( + mirror, + MirrorCookie::Expire, + "should not mirror a bidstream partner without mirror_enabled" + ); + assert!( + !serde_json::from_value::(json!({ + "name": "ids", "source_domain": "ids.example.com" + })) + .expect("should build partner") + .mirror_enabled, + "should default mirror_enabled to off" + ); + } + + #[test] + fn withholds_uids_whose_owner_lacks_vendor_consent() { + let entry = row(vec![uid("live", Expiry::Never)]); + let mut registry = partners(true); + assert!( + registry.mark_provider_owned("ids.example.com", NonZeroU16::new(9999)), + "should mark the source" + ); + let now = UnixSeconds::new(1000); + + let denied = build_mirror( + Some(&entry), + &consent_granting_vendors(vec![1]), + ®istry, + now, + ); + let granted = build_mirror( + Some(&entry), + &consent_granting_vendors(vec![9999]), + ®istry, + now, + ); + + assert_eq!( + denied, + MirrorCookie::Expire, + "should not mirror a UID whose owner's vendor consent is denied" + ); + assert!( + matches!(granted, MirrorCookie::Set(_)), + "should mirror the UID once the vendor has consent" + ); + } + #[test] fn does_not_mirror_browser_ingested_uid_of_pull_sync_source() { let mut entry = KvEntry::tombstone(1000); @@ -341,7 +416,7 @@ mod tests { .insert("ids.example.com".to_owned(), KvPartnerId::single("browser")); let pull_partner = serde_json::from_value(json!({ "name": "ids", "source_domain": "ids.example.com", "bidstream_enabled": true, - "pull_sync_enabled": true, "pull_sync_url": "https://sync.example.com/pull", + "mirror_enabled": true, "pull_sync_enabled": true, "pull_sync_url": "https://sync.example.com/pull", "pull_sync_allowed_domains": ["sync.example.com"], "ts_pull_token": "example-pull-token-32-bytes-minimum" })) diff --git a/crates/trusted-server-core/src/ec/prebid_eids.rs b/crates/trusted-server-core/src/ec/prebid_eids.rs index 8619fd0651..f0082eb9ac 100644 --- a/crates/trusted-server-core/src/ec/prebid_eids.rs +++ b/crates/trusted-server-core/src/ec/prebid_eids.rs @@ -355,6 +355,7 @@ mod tests { source_domain: source_domain.to_owned(), openrtb_atype: EcPartner::default_openrtb_atype(), bidstream_enabled: true, + mirror_enabled: false, api_token: Some(Redacted::new(format!( "token-{source_domain}-32-bytes-minimum-value" ))), diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index f9aa07ad8f..37722de3c9 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -8,6 +8,7 @@ use std::collections::{HashMap, HashSet}; use std::num::NonZeroU16; use error_stack::{Report, ResultExt as _}; +use http::HeaderValue; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::{ConsentContext, tcf_vendor_consent_denied}; @@ -34,6 +35,9 @@ pub struct PartnerConfig { pub openrtb_atype: i32, /// Whether this partner's UIDs appear in auction `user.eids`. pub bidstream_enabled: bool, + /// Whether this partner's provider-written UIDs are mirrored to the + /// JS-readable `__Host-ts-eids-mirror` cookie. + pub mirror_enabled: bool, /// SHA-256 hex of the partner's API token, when inbound API access is enabled. pub api_key_hash: Option, /// Max batch sync API requests per partner per minute. @@ -53,8 +57,8 @@ pub struct PartnerConfig { pub pull_sync_rate_limit: u32, /// Outbound bearer token for pull sync requests. pub ts_pull_token: Option>, - /// Whether an enabled identity provider claims this source. Browser - /// ingest skips it. + /// Whether an enabled identity provider, including the partner's own + /// pull sync, claims this source. pub provider_owned: bool, /// TCF vendor IDs of the configured providers, enabled or disabled, that /// claim this source. Its UIDs leave the server only while every one of @@ -64,11 +68,11 @@ pub struct PartnerConfig { /// source where GDPR applies. Its UIDs then never leave the server for a /// visitor whose jurisdiction is GDPR. pub excluded_where_gdpr_applies: bool, - /// Whether batch sync rejects this source's mappings: an enabled - /// configured provider owns it. A source owned only by its partner's pull - /// sync keeps accepting batch mappings, as it did before pull sync became - /// a provider. - pub batch_sync_rejected: bool, + /// Whether browser ingest and batch sync reject this source's IDs: an + /// enabled configured provider owns it. A source owned only by its + /// partner's pull sync keeps accepting both, as it did before pull sync + /// became a provider. + pub ingest_rejected: bool, } impl PartnerConfig { @@ -291,7 +295,7 @@ impl PartnerRegistry { } for source in &provider.sources { registry.mark_provider_owned(source, provider.tcf_vendor_id); - registry.reject_batch_sync(source); + registry.reject_ingest(source); } } // A disabled provider's stored UIDs stay behind its vendor consent, @@ -305,7 +309,7 @@ impl PartnerRegistry { for partner in ec .partners .iter() - .filter(|partner| pull_sync_registered && runs_pull_sync(partner)) + .filter(|partner| pull_sync_registered && partner.runs_pull_sync()) { registry.mark_provider_owned(&partner.source_domain, None); } @@ -396,37 +400,29 @@ impl PartnerRegistry { .is_some_and(|partner| partner.provider_owned) } - /// Marks `source` as owned by an enabled configured provider, so batch - /// sync rejects its mappings. Returns whether `source` is registered. - pub fn reject_batch_sync(&mut self, source: &str) -> bool { + /// Marks `source` as owned by an enabled configured provider, so browser + /// ingest and batch sync reject its IDs. Returns whether `source` is + /// registered. + pub fn reject_ingest(&mut self, source: &str) -> bool { let Ok(normalized) = normalize_partner_source_domain(source) else { return false; }; match self.by_source_domain.get_mut(&normalized) { Some(partner) => { - partner.batch_sync_rejected = true; + partner.ingest_rejected = true; true } None => false, } } - /// Whether batch sync rejects mappings for `source` (spec §8.5): an - /// enabled configured provider owns it. Pull-sync ownership alone does - /// not reject batch mappings. + /// Whether browser ingest and batch sync reject IDs for `source` + /// (spec §8.5): an enabled configured provider owns it. Pull-sync + /// ownership alone rejects neither. #[must_use] - pub fn rejects_batch_sync(&self, source: &str) -> bool { + pub fn rejects_ingest(&self, source: &str) -> bool { self.find_by_source_domain(source) - .is_some_and(|partner| partner.batch_sync_rejected) - } - - /// Returns all partners with `pull_sync_enabled = true`. - #[must_use] - pub fn pull_enabled_partners(&self) -> Vec<&PartnerConfig> { - self.by_source_domain - .values() - .filter(|p| p.pull_sync_enabled) - .collect() + .is_some_and(|partner| partner.ingest_rejected) } /// Partners in `[[ec.partners]]` order. @@ -457,12 +453,6 @@ impl PartnerRegistry { } } -/// Whether the partner runs pull sync as a provider, making its source -/// provider-owned. -fn runs_pull_sync(partner: &EcPartner) -> bool { - partner.pull_sync_enabled && partner.pull_sync_url.is_some() -} - fn validate_api_token( source_domain: &str, api_token: &str, @@ -496,6 +486,7 @@ fn build_partner_config( source_domain: normalized_source.to_owned(), openrtb_atype: partner.openrtb_atype, bidstream_enabled: partner.bidstream_enabled, + mirror_enabled: partner.mirror_enabled, api_key_hash: api_key_hash.map(ToOwned::to_owned), batch_rate_limit: partner.batch_rate_limit, pull_sync_enabled: partner.pull_sync_enabled, @@ -507,7 +498,7 @@ fn build_partner_config( provider_owned: false, owner_tcf_vendor_ids: Vec::new(), excluded_where_gdpr_applies: false, - batch_sync_rejected: false, + ingest_rejected: false, } } @@ -609,6 +600,17 @@ fn validate_pull_sync_fields( })); } + // The resolved token is sent as `Authorization: Bearer `; one that + // is not a valid header value would fail every provider build at startup. + if require_nonempty_token + && token_value + .is_some_and(|value| HeaderValue::try_from(format!("Bearer {value}")).is_err()) + { + return Err(Report::new(TrustedServerError::Configuration { + message: "ts_pull_token must be valid in an HTTP Authorization header".to_owned(), + })); + } + Ok(()) } @@ -627,6 +629,7 @@ mod tests { source_domain: source_domain.to_owned(), openrtb_atype: EcPartner::default_openrtb_atype(), bidstream_enabled: false, + mirror_enabled: false, api_token: Some(Redacted::new(api_token.to_owned())), batch_rate_limit: EcPartner::default_batch_rate_limit(), pull_sync_enabled: false, @@ -686,12 +689,12 @@ mod tests { "should mark pull-sync partner source" ); assert!( - registry.rejects_batch_sync("ids.example.com"), - "should reject batch sync for a configured provider's source" + registry.rejects_ingest("ids.example.com"), + "should reject browser and batch ingest for a configured provider's source" ); assert!( - !registry.rejects_batch_sync("pull.example.com"), - "should keep batch sync for a source owned only by pull sync" + !registry.rejects_ingest("pull.example.com"), + "should keep browser and batch ingest for a source owned only by pull sync" ); } @@ -940,7 +943,27 @@ mod tests { } #[test] - fn pull_enabled_partners_filters_correctly() { + fn from_config_rejects_a_pull_token_invalid_in_a_header() { + let mut pull_partner = make_partner("pull.example.com", &valid_api_token("token-p")); + pull_partner.pull_sync_enabled = true; + pull_partner.pull_sync_url = Some("https://pull.example.com/sync".to_owned()); + pull_partner.pull_sync_allowed_domains = vec!["pull.example.com".to_owned()]; + pull_partner.ts_pull_token = Some(Redacted::new("example\ntoken".to_owned())); + + let error = PartnerRegistry::from_config(std::slice::from_ref(&pull_partner)) + .expect_err("should reject a pull token that is not a valid header value"); + assert!( + format!("{error:?}").contains("Authorization header"), + "should name the header problem" + ); + + pull_partner.pull_sync_enabled = false; + PartnerRegistry::from_config(&[pull_partner]) + .expect("should not validate the pull token of a partner without pull sync"); + } + + #[test] + fn pull_sync_partner_keeps_its_token_without_an_api_token() { let mut pull_partner = make_partner("pull.example.com", &valid_api_token("token-p")); pull_partner.api_token = None; pull_partner.pull_sync_enabled = true; @@ -954,22 +977,19 @@ mod tests { ]; let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); - let pull_enabled = registry.pull_enabled_partners(); - assert_eq!( - pull_enabled.len(), - 1, - "should have exactly one pull-enabled partner" - ); - assert_eq!( - pull_enabled[0].source_domain, "pull.example.com", - "should be the correct partner" + let pull_partner = registry + .get("pull.example.com") + .expect("should register the pull-sync partner"); + assert!( + pull_partner.pull_sync_enabled, + "should keep pull sync enabled" ); assert!( - pull_enabled[0].api_key_hash.is_none(), + pull_partner.api_key_hash.is_none(), "should allow pull sync without an inbound API token" ); assert_eq!( - pull_enabled[0] + pull_partner .ts_pull_token .as_ref() .expect("should keep pull token") diff --git a/crates/trusted-server-core/src/ec/write_plan.rs b/crates/trusted-server-core/src/ec/write_plan.rs index c30e0e1cc0..072e6f9e61 100644 --- a/crates/trusted-server-core/src/ec/write_plan.rs +++ b/crates/trusted-server-core/src/ec/write_plan.rs @@ -97,7 +97,9 @@ pub fn plan_write(input: &WritePlanInput<'_>) -> Option { } } -/// Fills missing sources from the browser, skipping provider-owned ones. +/// Fills missing sources from the browser, skipping sources an enabled +/// configured provider owns. A source owned only by its partner's pull sync +/// is still filled, as it was before pull sync became a provider. /// /// Returns the sources it inserted, in insertion order. fn apply_browser_ingest( @@ -107,7 +109,7 @@ fn apply_browser_ingest( ) -> Vec { let mut ingested = Vec::new(); for update in updates { - if registry.is_provider_owned(&update.partner_id) { + if registry.rejects_ingest(&update.partner_id) { log::trace!( "EC write planner: skipping browser UID for provider-owned source '{}'", update.partner_id @@ -349,6 +351,7 @@ mod tests { registry.mark_provider_owned(source, None), "should mark {source}" ); + assert!(registry.reject_ingest(source), "should reject {source}"); } registry } @@ -926,6 +929,32 @@ mod tests { } } + #[test] + fn browser_ingest_fills_a_source_owned_only_by_pull_sync() { + // Arrange: pull sync owns the source but rejects no ingest. + let mut registry = registry(&[]); + assert!( + registry.mark_provider_owned("ids.example.com", None), + "should mark the pull-sync source" + ); + let snapshot = present(&live_entry()); + let ingest = ingest_owned(); + let lookups = failed_lookup(LookupStatus::Skipped(SkipReason::NotDue)); + + // Act + let change = plan_write(&input(&snapshot, None, &ingest, &lookups, ®istry)) + .expect("should plan an update"); + + // Assert + let RowChange::Update { entry, .. } = change else { + panic!("should update the existing row") + }; + assert_eq!( + entry.ids["ids.example.com"].uid, "browser-id", + "should store the browser UID for a pull-sync source" + ); + } + #[test] fn opted_out_source_is_not_refilled_from_the_browser_on_the_next_navigation() { // Arrange: the row after an opt-out holds nothing for the source. diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 30513ff1c7..0d384d3874 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -503,6 +503,11 @@ pub struct EcPartner { /// Whether this partner's UIDs appear in auction `user.eids`. #[serde(default, deserialize_with = "from_value_or_str")] pub bidstream_enabled: bool, + /// Whether this partner's provider-written UIDs are mirrored to the + /// JS-readable `__Host-ts-eids-mirror` cookie. Off by default; takes + /// effect only with `bidstream_enabled`. + #[serde(default, deserialize_with = "from_value_or_str")] + pub mirror_enabled: bool, /// Plaintext API token used by inbound batch sync and identify requests. /// /// When present, the token is hashed at startup for auth lookups. Omitting @@ -543,6 +548,13 @@ pub struct EcPartner { } impl EcPartner { + /// Whether this partner runs pull sync as an identity provider, which + /// makes its source provider-owned. + #[must_use] + pub fn runs_pull_sync(&self) -> bool { + self.pull_sync_enabled && self.pull_sync_url.is_some() + } + /// Known partner secret placeholders (`api_token` and `ts_pull_token`) that /// must not be used in deployments. pub const API_TOKEN_PLACEHOLDERS: &[&str] = &[ @@ -5161,6 +5173,7 @@ mod tests { source_domain: "canary-partner.example".to_string(), openrtb_atype: EcPartner::default_openrtb_atype(), bidstream_enabled: false, + mirror_enabled: false, api_token: Some(Redacted::new(CANARY_EC_PARTNER_API_TOKEN.to_string())), batch_rate_limit: EcPartner::default_batch_rate_limit(), pull_sync_enabled: false, @@ -7217,6 +7230,7 @@ source_domain = "partner.example.com" source_domain: "partner.example.com".to_owned(), openrtb_atype: EcPartner::default_openrtb_atype(), bidstream_enabled: false, + mirror_enabled: false, api_token: api_token.map(|token| Redacted::new(token.to_owned())), batch_rate_limit: EcPartner::default_batch_rate_limit(), pull_sync_enabled: true, diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index 7cda004904..a937932b85 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -131,14 +131,14 @@ pub fn build( /// One `pull-sync` provider per partner with pull sync enabled, built from the /// partner's existing fields with no config change. /// -/// Pull sync runs after the response has been sent, as it did before it -/// became a provider, so a slow partner never delays the page and a new EC -/// whose row does not exist yet is never pulled. Its hourly rate limit is -/// the partner's `pull_sync_rate_limit` (`None` when it is 0). +/// Pull sync runs after the response has been sent, so a slow partner never +/// delays the page and a new EC whose row does not exist yet is never pulled. +/// Its hourly rate limit is the partner's `pull_sync_rate_limit`, which +/// partner validation keeps nonzero. pub fn pull_sync_providers(ec: &Ec) -> impl Iterator + '_ { ec.partners .iter() - .filter(|partner| partner.pull_sync_enabled) + .filter(|partner| partner.runs_pull_sync()) .filter_map(|partner| { let endpoint = partner.pull_sync_url.clone()?; let mut config = IdentityProviderConfig::new( diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index c9caa6a6e6..7bd25c07d3 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -891,6 +891,7 @@ ec_store = "ec_identity_store" name = "Mocktioneer SSP" source_domain = "mocktioneer.example" bidstream_enabled = true +# mirror_enabled = true # mirror provider-written UIDs to a JS-readable cookie (default false) # api_token = "partner_api_token" # only for inbound identify or batch sync # ts_pull_token = "partner_ts_pull_token" # required when pull sync is enabled ``` diff --git a/docs/guide/ec-setup-guide.md b/docs/guide/ec-setup-guide.md index 1410e3deb3..932c65099a 100644 --- a/docs/guide/ec-setup-guide.md +++ b/docs/guide/ec-setup-guide.md @@ -43,8 +43,9 @@ do not call either API may omit it. Provision high-entropy values under [Configuration](/guide/configuration#secret-store-migration). A partner with pull sync enabled (`pull_sync_enabled` and `pull_sync_url`) -owns its source through the pull-sync provider, so browser EID ingestion skips -it. Batch sync keeps accepting its mappings, so a partner may use both. See +owns its source through the pull-sync provider. Browser EID ingestion and batch +sync keep filling that source, so a partner may use all three; pull sync only +fills a source that holds no UID. See [Pull Sync on Navigations](/guide/edge-cookies#pull-sync-on-navigations). Required behavior assumptions: diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index 17246fa13c..4eb73c19ce 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -265,7 +265,7 @@ Server-resolved EIDs and current-request Prebid EIDs are deduplicated by `source ### Pull Sync on Navigations -Pull sync runs as an identity provider on top-level page navigations, after the response has been sent, as it did before it became a provider. It uses the after-response budget (`identity_after_response_budget_ms`), so a slow partner never delays the page, and its result reaches the row in the follow-up write and is served from the next page load. Like other after-response lookups, it never creates a row. A pull-sync partner's source is provider-owned, so browser EID ingestion skips it. Batch sync still accepts the partner's mappings, as before. A source that already holds any live UID, including a batch-synced one, is not pulled again; a source with no UID is pulled at most `pull_sync_rate_limit` times per hour per EC. On Fastly that limit is enforced as a per-minute budget of at least one, so values below 60 behave like 60 per hour. At most `pull_sync_concurrency` partners are pulled per navigation (0 means no limit), unlike before, when every eligible partner was pulled in batches of that size. The partner order rotates every hour, so partners past the cap still get turns. +Pull sync runs as an identity provider after the response has been sent, on top-level page navigations from recognized browsers that send Fetch Metadata (`Sec-Fetch-Dest: document` and `Sec-Fetch-Mode: navigate`) and are not prefetches. Requests that fail the browser check never trigger it. Before it became a provider, it also ran after `/auction` responses and on navigations without Fetch Metadata. It uses the after-response budget (`identity_after_response_budget_ms`, shared with the other after-response providers), so a slow partner never delays the page; a partner that answers after the budget is not filled, where each pull used to wait up to 15 seconds for a first byte. Its result reaches the row in the follow-up write and is served from the next page load. Like other after-response lookups, it never creates a row. Browser EID ingestion (`ts-eids`, `sharedId`) still fills a pull-sync partner's source when it holds no UID, and batch sync still accepts the partner's mappings, as before. A source that already holds any live UID, including a browser-supplied or batch-synced one, is not pulled again; a source with no UID is pulled at most `pull_sync_rate_limit` times per hour per EC. On Fastly that limit is enforced as a per-minute budget of at least one, so values below 60 behave like 60 per hour. At most `pull_sync_concurrency` partners are pulled per navigation (0 means no limit), unlike before, when every eligible partner was pulled in batches of that size; a partner past the cap is logged as `skipped_concurrency_cap`. The partner order rotates every hour, so partners past the cap still get turns. When every pull-sync source is already held, the after-response phase makes no KV read. The former `ts-ec-pull-complete` completeness marker is retired. A response to a request that still carries it expires the leftover cookie. @@ -273,13 +273,14 @@ Explicit withdrawal expires `ts-ec` once the row is tombstoned or proven absent. ### EID Mirror Cookie -Trusted Server mirrors provider-written UIDs for `bidstream_enabled` partners into a JavaScript-readable cookie so client-side code can use them: +Trusted Server can mirror provider-written UIDs into a JavaScript-readable cookie so client-side code can use them. The mirror is off by default: a partner's UIDs are mirrored only when it sets both `bidstream_enabled = true` and `mirror_enabled = true`. - **Name**: `__Host-ts-eids-mirror` - **Attributes**: `Path=/; Secure; SameSite=Lax; Max-Age=`, host-only (no `Domain`) and not `HttpOnly` - **Contents**: unpadded base64url JSON `{"v":1,"eids":[{"source":"ids.example.com","uids":[{"id":"...","atype":1,"exp":1767225600}]}]}`, partners in `[[ec.partners]]` order, at most 3584 bytes; UIDs that do not fit are left out - **Contents source**: live UIDs written by an identity provider. Ownerless UIDs, from browser ingest, batch sync or legacy pull sync, are never mirrored. -- **Pull-sync UIDs**: pull sync now writes provider-owned UIDs, so a `bidstream_enabled` pull-sync partner's UIDs are mirrored. Before, they stayed in KV only; now any script on the page can read them, and the cookie is sent on every same-origin request. Set `bidstream_enabled = false` on a partner whose UIDs must not reach the page. +- **Pull-sync UIDs**: pull sync writes provider-owned UIDs, so a pull-sync partner with `mirror_enabled = true` has its UIDs mirrored. Any script on the page can read them, and the cookie is sent on every same-origin request, including to the publisher origin. Leave `mirror_enabled` off for a partner whose UIDs must not reach the page; its UIDs still reach the bidstream when `bidstream_enabled` is set. +- **Vendor consent**: a UID whose owning provider has a `tcf_vendor_id` is left out while the request's TCF string denies that vendor, as it is from the bidstream. - **Updates** (best effort): page navigations rebuild it from the row after that navigation's write. It is expired only when the request carried it, either because no live provider UID qualifies or because EC and EID consent is missing (on any response). Responses to browsers without a mirror get no mirror `Set-Cookie`. When the identity row cannot be read (KV failure) the mirror is left untouched, and a mirror that cannot be built is skipped without failing the response. Readers must drop UIDs whose `exp` (Unix seconds) has passed and strip `exp` before using a UID. diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 81c2cb939a..ea76384fef 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -114,6 +114,9 @@ pull_sync_concurrency = 3 # openrtb_atype = 3 # include this partner's UIDs in auction user.eids # bidstream_enabled = true +# also mirror provider-written UIDs to the JS-readable __Host-ts-eids-mirror +# cookie (default false; requires bidstream_enabled) +# mirror_enabled = false # Only for inbound identify or batch-sync API access: # api_token = "partner_api_token" # Required when pull sync is enabled: From 9de1d0454ee2fa4240e833510080fabd52bee761 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 02:35:52 +0530 Subject: [PATCH 15/20] Address the third review of pull sync and the EID mirror Mirror: expire a carried mirror where it cannot be rebuilt (non-navigations, or a navigation without a readable row) when the request denies TCF vendor consent to a mirrored partner's owning provider; reject mirror_enabled without bidstream_enabled at startup and at config push; omit a default mirror_enabled when serializing partner config so older binaries still load the blob. Registry: restore provider_owned to its base meaning (an enabled configured provider claims the source) and drop the duplicate ingest_rejected flag; record runs_pull_sync on PartnerConfig so the double-claim check is a single lookup. Cleanup: drop the unused log dependency of the identity-providers crate, restore holds_live_uid's doc line, assert the exact pull-token error. Docs: align the design spec with the mirror and pull sync as shipped, add CHANGELOG entries including the breaking pull-token validation, and note the row read the retired marker used to skip. Signed-off-by: dhruv8sh --- CHANGELOG.md | 3 + crates/trusted-server-core/src/config.rs | 17 +-- .../trusted-server-core/src/ec/batch_sync.rs | 12 +- .../src/ec/coordinator/mod.rs | 2 +- crates/trusted-server-core/src/ec/finalize.rs | 113 +++++++++++++++-- crates/trusted-server-core/src/ec/mirror.rs | 6 +- crates/trusted-server-core/src/ec/registry.rs | 119 ++++++++++-------- .../trusted-server-core/src/ec/write_plan.rs | 11 +- crates/trusted-server-core/src/settings.rs | 41 +++++- docs/guide/edge-cookies.md | 6 +- .../2026-10-30-s2s-eid-collection-design.md | 91 +++++++------- 11 files changed, 280 insertions(+), 141 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38862aa663..704f72d79c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 the host can surface bytes that already arrived, and reads on into the body of a response whose headers arrived by the cutoff instead of abandoning it. - Added server-side identity providers (`[[ec.identity_providers]]`, Fastly only). Enabled providers look up a visitor's IDs on publisher navigations within `ec.identity_budget_ms` (1–1000, default 200), or after the response within `ec.identity_after_response_budget_ms` (1–5000, default 2000) for `collect = "after_response"`. A provider owns its `sources`: batch sync rejects them with the new `provider_owned` reason, pull sync skips them, and browser IDs never fill them. Set `tcf_vendor_id` to require TCF vendor consent for lookups and for every ID of the provider's sources in bidstream, `x-ts-eids` and `/identify`, browser-sent IDs included; a disabled provider keeps that gate. See [Identity providers](docs/guide/configuration.md#identity-providers). +- Pull sync now runs as a `pull-sync` identity provider built from the existing `[[ec.partners]]` settings, after the response on top-level publisher navigations from recognized browsers, within `ec.identity_after_response_budget_ms` shared with other after-response providers. It no longer runs after `/auction` responses, on prefetches or on navigations without Fetch Metadata, and a partner slower than the budget is not filled, where each pull used to wait up to 15 seconds. `ec.pull_sync_concurrency` now caps the partners pulled per navigation (`0` means no limit; it used to mean one at a time), partner order rotates hourly, and a partner past the cap logs `skipped_concurrency_cap`. The `ts-ec-pull-complete` marker is retired and leftover cookies are expired; without it, a returning navigation reads the EC row before the origin response even when nothing else needs it. `kind = "pull-sync"` in `[[ec.identity_providers]]` is rejected, as is a configured provider that claims a pull-sync partner's source. +- Added `mirror_enabled` on `[[ec.partners]]` (default `false`; requires `bidstream_enabled`). The partner's provider-written UIDs are mirrored to the JavaScript-readable `__Host-ts-eids-mirror` cookie, leaving out UIDs whose owning provider lacks TCF vendor consent. See [EID Mirror Cookie](docs/guide/edge-cookies.md#eid-mirror-cookie). +- **Breaking:** For a partner with pull sync enabled, a resolved `ts_pull_token` that cannot form an `Authorization: Bearer` header, such as one ending in a newline, now fails config validation and stops startup. Before, only that partner's pull sync failed, after the page was sent. Check pull-token secrets before deploying. - Batch sync reports a new `row_full` item reason when a mapping would push the EC row past its new 16 KiB size limit, and continues with the remaining groups. - An EC row over 16 KiB written by an earlier release loses its largest partner records on its next write that adds an ID, so the added ID still lands. - **Breaking:** An EC is now minted, and the EC row written, only on a top-level publisher navigation: a `GET` with `Sec-Fetch-Dest: document`, `Sec-Fetch-Mode: navigate` and no `prefetch` in `Sec-Purpose`. Clients that send no Fetch Metadata get no EC, a navigation answered with a 301, 302, 303, 307 or 308 redirect mints none, integration proxies no longer mint one, and prefetches and prerenders never mint or write. Each navigation makes at most one conditional page-load write; a consent withdrawal's tombstone is still written on any request that carries the EC cookie. diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 39943d5516..504fc3b973 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -478,17 +478,9 @@ fn validate_identity_providers(ec: &Ec) -> Result<(), Report // fail the coordinator build on every request. for config in ec.identity_providers.iter().filter(|config| config.enabled) { if let Some(source) = config.sources.iter().find(|source| { - let claimed = partners + partners .find_by_source_domain(source) - .map(|partner| partner.source_domain.as_str()); - claimed.is_some() - && ec.partners.iter().any(|partner| { - partner.runs_pull_sync() - && partners - .find_by_source_domain(&partner.source_domain) - .map(|config| config.source_domain.as_str()) - == claimed - }) + .is_some_and(|partner| partner.runs_pull_sync) }) { return Err(Report::new(TrustedServerError::Configuration { message: format!( @@ -1768,9 +1760,8 @@ gam_network_id = "99999" .expect_err("should reject a pull token that cannot form a header"); assert!( - err.to_string().contains("pull sync config invalid") - || format!("{err:?}").contains("Authorization header"), - "should explain the invalid pull token: {err:?}" + format!("{err:?}").contains("Authorization header"), + "should name the Authorization header problem: {err:?}" ); } diff --git a/crates/trusted-server-core/src/ec/batch_sync.rs b/crates/trusted-server-core/src/ec/batch_sync.rs index 19f5f45f72..df2e067709 100644 --- a/crates/trusted-server-core/src/ec/batch_sync.rs +++ b/crates/trusted-server-core/src/ec/batch_sync.rs @@ -172,7 +172,7 @@ fn handle_batch_sync_with_writer( let (accepted, errors) = process_mappings_for_partner( writer, &partner.source_domain, - registry.rejects_ingest(&partner.source_domain), + registry.is_provider_owned(&partner.source_domain), &body.mappings, ); @@ -995,12 +995,8 @@ mod tests { #[test] fn handle_batch_sync_accepts_mappings_for_a_pull_sync_owned_source() { - // Arrange: pull sync owns the source, but no configured provider does. - let mut registry = test_registry(); - assert!( - registry.mark_provider_owned("ssp.example.com", None), - "should mark the pull-sync source" - ); + // Arrange: pull sync runs for the source, but no configured provider owns it. + let registry = test_registry(); let limiter = MockRateLimiter { should_exceed: false, }; @@ -1035,7 +1031,7 @@ mod tests { fn handle_batch_sync_rejects_mappings_for_a_configured_provider_source() { let mut registry = test_registry(); assert!( - registry.reject_ingest("ssp.example.com"), + registry.mark_provider_owned("ssp.example.com", None), "should mark the configured provider source" ); let limiter = MockRateLimiter { diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 85f3c3c640..7c47df6de2 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -318,7 +318,6 @@ impl ProviderCoordinator

{ }) } - /// Whether the row holds a live provider-written UID for `source`. /// Whether any provider has a due source in `entry`, ignoring consent and /// rate limits, so a caller can skip a row read that cannot lead to a /// lookup. `None` (no row known) counts every source as due. @@ -330,6 +329,7 @@ impl ProviderCoordinator

{ }) } + /// Whether the row holds a live provider-written UID for `source`. fn holds_live_uid(entry: Option<&KvEntry>, source: &str, now: UnixSeconds) -> bool { entry .and_then(|entry| entry.ids.get(source)) diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 9f121e1795..d287462d62 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -12,7 +12,7 @@ use edgezero_core::body::Body as EdgeBody; use http::Response; use super::consent::{ec_consent_granted, ec_consent_withdrawn}; -use crate::consent::{ConsentContext, allows_eid_persistence}; +use crate::consent::{ConsentContext, allows_eid_persistence, tcf_vendor_consent_denied}; use crate::constants::{COOKIE_TS_EC_PULL_COMPLETE, COOKIE_TS_EIDS_MIRROR}; use crate::settings::Settings; @@ -90,6 +90,10 @@ pub fn ec_finalize_response( /// cacheability. An unchanged mirror is therefore not re-extended; once it /// lapses, the next navigation sets it again. When the row could not be read /// authoritatively (KV failure), the mirror is left untouched. +/// +/// Where the mirror cannot be rebuilt (non-navigations, or a navigation +/// without a readable row), a carried mirror is expired when the request +/// denies TCF vendor consent to the owning provider of any mirrored partner. fn apply_mirror( ec_context: &EcContext, registry: &PartnerRegistry, @@ -98,18 +102,21 @@ fn apply_mirror( ) { let decision = if !mirror_consent(ec_context.consent()) { MirrorCookie::Expire - } else if navigation { - let Some(row) = mirror_row(ec_context) else { - log::debug!("Leaving the mirror cookie untouched: identity row is unavailable"); - return; - }; + } else if navigation && let Some(row) = mirror_row(ec_context) { build_mirror( row, ec_context.consent(), registry, UnixSeconds::new(current_timestamp()), ) + } else if mirror_vendor_denied(ec_context, registry) { + // The mirror cannot be rebuilt here, and it may hold UIDs of a + // provider whose TCF vendor consent is now denied. + MirrorCookie::Expire } else { + if navigation { + log::debug!("Leaving the mirror cookie untouched: identity row is unavailable"); + } return; }; match decision { @@ -131,6 +138,20 @@ fn apply_mirror( } } +/// Whether the request carries a mirror and denies TCF vendor consent to the +/// owning provider of a mirrored partner, so the mirror may hold UIDs that +/// must no longer reach the page. +fn mirror_vendor_denied(ec_context: &EcContext, registry: &PartnerRegistry) -> bool { + ec_context.mirror_cookie_present() + && registry.in_config_order().any(|partner| { + partner.bidstream_enabled + && partner.mirror_enabled + && partner + .owner_tcf_vendor_id + .is_some_and(|vendor| tcf_vendor_consent_denied(ec_context.consent(), vendor)) + }) +} + /// The row the mirror is built from. /// /// Returns `Some(None)` when there is authoritatively no row (no EC or a @@ -646,6 +667,8 @@ mod tests { use base64::Engine as _; + use std::num::NonZeroU16; + use super::*; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ConsentContext, ConsentSource}; @@ -659,6 +682,7 @@ mod tests { use crate::openrtb::{Eid, Uid}; use crate::redacted::Redacted; use crate::settings::EcPartner; + use crate::test_support::tests::consent_granting_vendors; use crate::test_support::tests::{create_test_settings, purpose_one_only_consent}; fn empty_response() -> Response { @@ -1617,10 +1641,6 @@ mod tests { registry.mark_provider_owned("sharedid.org", None), "should mark the provider source" ); - assert!( - registry.reject_ingest("sharedid.org"), - "should reject browser ingest for the configured provider's source" - ); let mut response = empty_response(); // Act @@ -2031,6 +2051,19 @@ mod tests { mirror: Option<&str>, kv_read_failed: bool, navigation: bool, + ) -> Response { + run_mirror_request_with_owner(lookups, mirror, kv_read_failed, navigation, false) + } + + /// Runs a mirror request. With `vendor_denied`, the source's owning + /// provider has TCF vendor 9999 and the request's TCF string keeps + /// Purposes 1 and 4 but denies that vendor. + fn run_mirror_request_with_owner( + lookups: LookupResults, + mirror: Option<&str>, + kv_read_failed: bool, + navigation: bool, + vendor_denied: bool, ) -> Response { let settings = create_test_settings(); let ec_id = sample_ec_id("mirr01"); @@ -2057,7 +2090,13 @@ mod tests { let mut partner = make_partner("ids.example.com"); partner.mirror_enabled = true; let mut registry = PartnerRegistry::from_config(&[partner]).expect("should build registry"); - registry.mark_provider_owned("ids.example.com", None); + let owner_vendor = if vendor_denied { + *ec_context.consent_mut() = consent_granting_vendors(Vec::new()); + NonZeroU16::new(9999) + } else { + None + }; + registry.mark_provider_owned("ids.example.com", owner_vendor); let mut response = empty_response(); ec_finalize_response( @@ -2103,6 +2142,58 @@ mod tests { ); } + #[test] + fn vendor_withdrawal_expires_the_mirror_on_a_non_navigation() { + let response = run_mirror_request_with_owner( + provider_found("ids.example.com"), + Some("stale"), + false, + false, + true, + ); + + let cookies = mirror_cookies(&response); + assert_eq!(cookies.len(), 1, "should emit one mirror cookie"); + assert!( + cookies[0].contains("Max-Age=0"), + "should expire a mirror whose owning vendor lost consent" + ); + } + + #[test] + fn vendor_withdrawal_expires_the_mirror_when_the_row_is_unavailable() { + let response = run_mirror_request_with_owner( + provider_found("ids.example.com"), + Some("stale"), + true, + true, + true, + ); + + let cookies = mirror_cookies(&response); + assert_eq!(cookies.len(), 1, "should emit one mirror cookie"); + assert!( + cookies[0].contains("Max-Age=0"), + "should expire the mirror without a readable row" + ); + } + + #[test] + fn vendor_withdrawal_without_a_mirror_cookie_emits_nothing() { + let response = run_mirror_request_with_owner( + provider_found("ids.example.com"), + None, + false, + false, + true, + ); + + assert!( + mirror_cookies(&response).is_empty(), + "should not expire a mirror the request did not carry" + ); + } + #[test] fn consented_non_navigation_leaves_the_mirror_untouched() { let response = run_mirror_request( diff --git a/crates/trusted-server-core/src/ec/mirror.rs b/crates/trusted-server-core/src/ec/mirror.rs index 3cf82746eb..be7a933239 100644 --- a/crates/trusted-server-core/src/ec/mirror.rs +++ b/crates/trusted-server-core/src/ec/mirror.rs @@ -333,7 +333,7 @@ mod tests { ); assert!( matches!( - build_mirror(Some(&entry), &granting(), &partners(false), now), + build_mirror(Some(&entry), &granting(), &partners_with(false, false), now), MirrorCookie::Expire ), "should expire for non-bidstream sources" @@ -421,9 +421,7 @@ mod tests { "ts_pull_token": "example-pull-token-32-bytes-minimum" })) .expect("should build partner"); - let mut pull = - PartnerRegistry::from_config(&[pull_partner]).expect("should build registry"); - pull.mark_provider_owned("ids.example.com", None); + let pull = PartnerRegistry::from_config(&[pull_partner]).expect("should build registry"); let mirror = build_mirror(Some(&entry), &granting(), &pull, UnixSeconds::new(1000)); diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index 37722de3c9..e42d882d8e 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -57,8 +57,12 @@ pub struct PartnerConfig { pub pull_sync_rate_limit: u32, /// Outbound bearer token for pull sync requests. pub ts_pull_token: Option>, - /// Whether an enabled identity provider, including the partner's own - /// pull sync, claims this source. + /// Whether the partner runs pull sync as an identity provider + /// ([`EcPartner::runs_pull_sync`]). Browser ingest and batch sync still + /// fill its source. + pub runs_pull_sync: bool, + /// Whether an enabled configured identity provider claims this source. + /// Browser ingest skips it and batch sync rejects it. pub provider_owned: bool, /// TCF vendor IDs of the configured providers, enabled or disabled, that /// claim this source. Its UIDs leave the server only while every one of @@ -68,11 +72,6 @@ pub struct PartnerConfig { /// source where GDPR applies. Its UIDs then never leave the server for a /// visitor whose jurisdiction is GDPR. pub excluded_where_gdpr_applies: bool, - /// Whether browser ingest and batch sync reject this source's IDs: an - /// enabled configured provider owns it. A source owned only by its - /// partner's pull sync keeps accepting both, as it did before pull sync - /// became a provider. - pub ingest_rejected: bool, } impl PartnerConfig { @@ -158,6 +157,8 @@ impl PartnerRegistry { }) })?; + validate_mirror_requires_bidstream(partner, &normalized_source)?; + if partner.pull_sync_enabled { validate_pull_sync_fields( partner.pull_sync_url.as_deref(), @@ -222,6 +223,8 @@ impl PartnerRegistry { None }; + validate_mirror_requires_bidstream(partner, &normalized_source)?; + let config = build_partner_config(partner, &normalized_source, api_key_hash.as_deref()); validate_rate_limits(&config).change_context(TrustedServerError::Configuration { @@ -269,8 +272,8 @@ impl PartnerRegistry { } /// Builds the registry from `[ec]` settings and marks every source claimed - /// by an enabled identity provider, and every pull-sync partner when the - /// `pull-sync` kind is registered, as provider-owned. + /// by an enabled identity provider whose kind is registered as + /// provider-owned. /// /// # Errors /// @@ -295,7 +298,6 @@ impl PartnerRegistry { } for source in &provider.sources { registry.mark_provider_owned(source, provider.tcf_vendor_id); - registry.reject_ingest(source); } } // A disabled provider's stored UIDs stay behind its vendor consent, @@ -305,14 +307,6 @@ impl PartnerRegistry { registry.keep_vendor_gate(source, provider.tcf_vendor_id); } } - let pull_sync_registered = is_registered_kind(PULL_SYNC_KIND); - for partner in ec - .partners - .iter() - .filter(|partner| pull_sync_registered && partner.runs_pull_sync()) - { - registry.mark_provider_owned(&partner.source_domain, None); - } Ok(registry) } @@ -400,31 +394,6 @@ impl PartnerRegistry { .is_some_and(|partner| partner.provider_owned) } - /// Marks `source` as owned by an enabled configured provider, so browser - /// ingest and batch sync reject its IDs. Returns whether `source` is - /// registered. - pub fn reject_ingest(&mut self, source: &str) -> bool { - let Ok(normalized) = normalize_partner_source_domain(source) else { - return false; - }; - match self.by_source_domain.get_mut(&normalized) { - Some(partner) => { - partner.ingest_rejected = true; - true - } - None => false, - } - } - - /// Whether browser ingest and batch sync reject IDs for `source` - /// (spec §8.5): an enabled configured provider owns it. Pull-sync - /// ownership alone rejects neither. - #[must_use] - pub fn rejects_ingest(&self, source: &str) -> bool { - self.find_by_source_domain(source) - .is_some_and(|partner| partner.ingest_rejected) - } - /// Partners in `[[ec.partners]]` order. pub fn in_config_order(&self) -> impl Iterator { self.order @@ -495,13 +464,29 @@ fn build_partner_config( pull_sync_ttl_sec: partner.pull_sync_ttl_sec, pull_sync_rate_limit: partner.pull_sync_rate_limit, ts_pull_token: partner.ts_pull_token.clone(), + runs_pull_sync: partner.runs_pull_sync(), provider_owned: false, owner_tcf_vendor_ids: Vec::new(), excluded_where_gdpr_applies: false, - ingest_rejected: false, } } +/// Rejects `mirror_enabled` without `bidstream_enabled`: the mirror only +/// carries bidstream UIDs, so the flag alone would silently do nothing. +fn validate_mirror_requires_bidstream( + partner: &EcPartner, + normalized_source: &str, +) -> Result<(), Report> { + if partner.mirror_enabled && !partner.bidstream_enabled { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "ec.partners: '{normalized_source}' sets mirror_enabled without bidstream_enabled" + ), + })); + } + Ok(()) +} + fn validate_rate_limits(config: &PartnerConfig) -> Result<(), Report> { validate_rate_limits_values(config.batch_rate_limit, config.pull_sync_rate_limit).map_err( |message| { @@ -685,16 +670,20 @@ mod tests { "should keep the disabled provider's vendor consent gate" ); assert!( - registry.is_provider_owned("pull.example.com"), - "should mark pull-sync partner source" + !registry.is_provider_owned("pull.example.com"), + "should keep browser and batch ingest for a source owned only by pull sync" ); assert!( - registry.rejects_ingest("ids.example.com"), - "should reject browser and batch ingest for a configured provider's source" + registry + .get("pull.example.com") + .is_some_and(|partner| partner.runs_pull_sync), + "should record that the partner runs pull sync" ); assert!( - !registry.rejects_ingest("pull.example.com"), - "should keep browser and batch ingest for a source owned only by pull sync" + !registry + .get("ids.example.com") + .is_some_and(|partner| partner.runs_pull_sync), + "should not record pull sync for a partner without it" ); } @@ -1051,6 +1040,36 @@ mod tests { ); } + #[test] + fn mirror_without_bidstream_is_rejected() { + let mut partner = make_partner("ids.example.com", &valid_api_token("token-m")); + partner.mirror_enabled = true; + + let runtime = PartnerRegistry::from_config(std::slice::from_ref(&partner)) + .expect_err("should reject mirror_enabled without bidstream_enabled at startup"); + let deploy = PartnerRegistry::validate_config_for_deploy(std::slice::from_ref(&partner)) + .expect_err("should reject mirror_enabled without bidstream_enabled at push"); + + for error in [runtime, deploy] { + assert!( + format!("{error:?}").contains("mirror_enabled without bidstream_enabled"), + "should name the misconfiguration: {error:?}" + ); + } + } + + #[test] + fn mirror_with_bidstream_is_accepted() { + let mut partner = make_partner("ids.example.com", &valid_api_token("token-m")); + partner.mirror_enabled = true; + partner.bidstream_enabled = true; + + PartnerRegistry::from_config(std::slice::from_ref(&partner)) + .expect("should accept mirror_enabled with bidstream_enabled"); + PartnerRegistry::validate_config_for_deploy(std::slice::from_ref(&partner)) + .expect("should accept mirror_enabled with bidstream_enabled at push"); + } + #[test] fn zero_batch_rate_limit_is_rejected() { let mut partner = make_partner("ssp.example.com", &valid_api_token("token-a")); diff --git a/crates/trusted-server-core/src/ec/write_plan.rs b/crates/trusted-server-core/src/ec/write_plan.rs index 072e6f9e61..f83f108594 100644 --- a/crates/trusted-server-core/src/ec/write_plan.rs +++ b/crates/trusted-server-core/src/ec/write_plan.rs @@ -109,7 +109,7 @@ fn apply_browser_ingest( ) -> Vec { let mut ingested = Vec::new(); for update in updates { - if registry.rejects_ingest(&update.partner_id) { + if registry.is_provider_owned(&update.partner_id) { log::trace!( "EC write planner: skipping browser UID for provider-owned source '{}'", update.partner_id @@ -351,7 +351,6 @@ mod tests { registry.mark_provider_owned(source, None), "should mark {source}" ); - assert!(registry.reject_ingest(source), "should reject {source}"); } registry } @@ -931,12 +930,8 @@ mod tests { #[test] fn browser_ingest_fills_a_source_owned_only_by_pull_sync() { - // Arrange: pull sync owns the source but rejects no ingest. - let mut registry = registry(&[]); - assert!( - registry.mark_provider_owned("ids.example.com", None), - "should mark the pull-sync source" - ); + // Arrange: pull sync runs for the source, but no configured provider owns it. + let registry = registry(&[]); let snapshot = present(&live_entry()); let ingest = ingest_owned(); let lookups = failed_lookup(LookupStatus::Skipped(SkipReason::NotDue)); diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 0d384d3874..30d8bd8959 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -506,7 +506,11 @@ pub struct EcPartner { /// Whether this partner's provider-written UIDs are mirrored to the /// JS-readable `__Host-ts-eids-mirror` cookie. Off by default; takes /// effect only with `bidstream_enabled`. - #[serde(default, deserialize_with = "from_value_or_str")] + #[serde( + default, + skip_serializing_if = "is_false", + deserialize_with = "from_value_or_str" + )] pub mirror_enabled: bool, /// Plaintext API token used by inbound batch sync and identify requests. /// @@ -4753,6 +4757,41 @@ mod tests { ); } + #[test] + fn default_partner_omits_mirror_enabled_for_rollback() { + let partner: EcPartner = serde_json::from_value(json!({ + "name": "ids", + "source_domain": "ids.example.com", + })) + .expect("should deserialize partner"); + + let serialized = serde_json::to_value(&partner).expect("should serialize partner"); + + assert!( + serialized.get("mirror_enabled").is_none(), + "a default partner must not carry the field, or an older binary rejects it: {serialized}" + ); + } + + #[test] + fn enabled_mirror_serializes_and_round_trips() { + let partner: EcPartner = serde_json::from_value(json!({ + "name": "ids", + "source_domain": "ids.example.com", + "mirror_enabled": true, + })) + .expect("should deserialize partner"); + + let serialized = serde_json::to_value(&partner).expect("should serialize partner"); + let restored: EcPartner = + serde_json::from_value(serialized).expect("should deserialize serialized partner"); + + assert!( + restored.mirror_enabled, + "an enabled mirror must survive a round trip" + ); + } + #[test] fn enabled_sandbox_metrics_serializes_and_round_trips() { let config = DebugConfig { diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index 4eb73c19ce..09613f1c7b 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -267,13 +267,13 @@ Server-resolved EIDs and current-request Prebid EIDs are deduplicated by `source Pull sync runs as an identity provider after the response has been sent, on top-level page navigations from recognized browsers that send Fetch Metadata (`Sec-Fetch-Dest: document` and `Sec-Fetch-Mode: navigate`) and are not prefetches. Requests that fail the browser check never trigger it. Before it became a provider, it also ran after `/auction` responses and on navigations without Fetch Metadata. It uses the after-response budget (`identity_after_response_budget_ms`, shared with the other after-response providers), so a slow partner never delays the page; a partner that answers after the budget is not filled, where each pull used to wait up to 15 seconds for a first byte. Its result reaches the row in the follow-up write and is served from the next page load. Like other after-response lookups, it never creates a row. Browser EID ingestion (`ts-eids`, `sharedId`) still fills a pull-sync partner's source when it holds no UID, and batch sync still accepts the partner's mappings, as before. A source that already holds any live UID, including a browser-supplied or batch-synced one, is not pulled again; a source with no UID is pulled at most `pull_sync_rate_limit` times per hour per EC. On Fastly that limit is enforced as a per-minute budget of at least one, so values below 60 behave like 60 per hour. At most `pull_sync_concurrency` partners are pulled per navigation (0 means no limit), unlike before, when every eligible partner was pulled in batches of that size; a partner past the cap is logged as `skipped_concurrency_cap`. The partner order rotates every hour, so partners past the cap still get turns. When every pull-sync source is already held, the after-response phase makes no KV read. -The former `ts-ec-pull-complete` completeness marker is retired. A response to a request that still carries it expires the leftover cookie. +The former `ts-ec-pull-complete` completeness marker is retired. A response to a request that still carries it expires the leftover cookie. The marker used to let a returning navigation skip reading the EC row before the origin response when no auction, browser EID cookie or withdrawal needed it; that navigation now reads the row. With the default inline mode the read overlaps the origin fetch. Explicit withdrawal expires `ts-ec` once the row is tombstoned or proven absent. While the tombstone cannot be written (KV unavailable, CAS retries exhausted, or a row that cannot be read), `ts-ec` is kept so the next request that carries it retries the withdrawal. ### EID Mirror Cookie -Trusted Server can mirror provider-written UIDs into a JavaScript-readable cookie so client-side code can use them. The mirror is off by default: a partner's UIDs are mirrored only when it sets both `bidstream_enabled = true` and `mirror_enabled = true`. +Trusted Server can mirror provider-written UIDs into a JavaScript-readable cookie so client-side code can use them. The mirror is off by default: a partner's UIDs are mirrored only when it sets both `bidstream_enabled = true` and `mirror_enabled = true`. Setting `mirror_enabled` without `bidstream_enabled` is rejected at startup and by `ts config push`. - **Name**: `__Host-ts-eids-mirror` - **Attributes**: `Path=/; Secure; SameSite=Lax; Max-Age=`, host-only (no `Domain`) and not `HttpOnly` @@ -281,7 +281,7 @@ Trusted Server can mirror provider-written UIDs into a JavaScript-readable cooki - **Contents source**: live UIDs written by an identity provider. Ownerless UIDs, from browser ingest, batch sync or legacy pull sync, are never mirrored. - **Pull-sync UIDs**: pull sync writes provider-owned UIDs, so a pull-sync partner with `mirror_enabled = true` has its UIDs mirrored. Any script on the page can read them, and the cookie is sent on every same-origin request, including to the publisher origin. Leave `mirror_enabled` off for a partner whose UIDs must not reach the page; its UIDs still reach the bidstream when `bidstream_enabled` is set. - **Vendor consent**: a UID whose owning provider has a `tcf_vendor_id` is left out while the request's TCF string denies that vendor, as it is from the bidstream. -- **Updates** (best effort): page navigations rebuild it from the row after that navigation's write. It is expired only when the request carried it, either because no live provider UID qualifies or because EC and EID consent is missing (on any response). Responses to browsers without a mirror get no mirror `Set-Cookie`. When the identity row cannot be read (KV failure) the mirror is left untouched, and a mirror that cannot be built is skipped without failing the response. +- **Updates** (best effort): page navigations rebuild it from the row after that navigation's write. It is expired only when the request carried it, either because no live provider UID qualifies or because EC and EID consent is missing (on any response). Where it cannot be rebuilt (other responses, or a navigation whose row cannot be read), it is also expired when the request denies TCF vendor consent to the owning provider of a mirrored partner. Responses to browsers without a mirror get no mirror `Set-Cookie`. When the identity row cannot be read (KV failure) the mirror is left untouched, and a mirror that cannot be built is skipped without failing the response. Readers must drop UIDs whose `exp` (Unix seconds) has passed and strip `exp` before using a UID. diff --git a/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md b/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md index 936574a10a..d9b1259c00 100644 --- a/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md +++ b/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md @@ -72,22 +72,22 @@ decision. These were agreed before drafting and are binding on the rest of the spec. -| # | Topic | Decision | -| --- | ------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| D1 | Which request may write | Only the top-level document navigation. `/auction`, `/_ts/page-bids` and integration-proxy requests never write KV. | -| D2 | Provider timing | Provider requests start when the navigation starts, run concurrently with the origin fetch, and are collected within a shared total budget of about 200 ms. Their results join the navigation's write. | -| D3 | Which navigations look up | Any navigation where an enabled provider has a due source (missing or expired UID). No-match results are not cached in V1. | -| D4 | Failed write | Not retried in that page load. KV and the mirror cookie stay unchanged. A new EC is not issued if its row was not created. The next navigation is a new page load and may write. | -| D5 | Availability | Provider IDs are guaranteed to `/auction` from the next page load. On the same page they are used if the write already completed. | -| D6 | Mirror cookie | JavaScript-readable. Holds provider-owned UIDs of `bidstream_enabled` sources only. Never trusted by the server over KV. Filled deterministically up to its size limit. | -| D6b | Mirror consumer | The tsjs change that reads the mirror and hands IDs to Prebid is a separate optional PR. | -| D7 | Expiry | Declared by the writing provider per UID. A UID without `exp` is served until replaced or removed. Browser and batch IDs carry no `exp` and no owner. | -| D8 | Conflicts | On a provider-owned source, the provider's UIDs replace that source's UIDs. Browser ingest fills only missing sources and never overwrites provider-owned UIDs. Batch sync rejects provider-owned sources. | -| D9 | Removal | Only consent withdrawal (the existing whole-row tombstone) is in scope. A tombstone wins over concurrent and later collection; IDs return only after consent is granted again. | -| D10 | Pull sync | Re-expressed as the first provider on the trait; the separate post-send dispatcher is removed. | -| D11 | Consent | Existing EID-persistence consent check, plus an optional per-provider TCF vendor ID that must also have vendor consent. | -| D12 | KV schema | Stay on `v: 1` with additive optional fields; no migration. | -| D13 | Other consumers | Provider IDs appear in `x-ts-eids` and `/_ts/api/v1/identify`, subject to `bidstream_enabled`, expiry and consent. | +| # | Topic | Decision | +| --- | ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| D1 | Which request may write | Only the top-level document navigation. `/auction`, `/_ts/page-bids` and integration-proxy requests never write KV. | +| D2 | Provider timing | Provider requests start when the navigation starts, run concurrently with the origin fetch, and are collected within a shared total budget of about 200 ms. Their results join the navigation's write. | +| D3 | Which navigations look up | Any navigation where an enabled provider has a due source (missing or expired UID). No-match results are not cached in V1. | +| D4 | Failed write | Not retried in that page load. KV and the mirror cookie stay unchanged. A new EC is not issued if its row was not created. The next navigation is a new page load and may write. | +| D5 | Availability | Provider IDs are guaranteed to `/auction` from the next page load. On the same page they are used if the write already completed. | +| D6 | Mirror cookie | JavaScript-readable. Holds provider-owned UIDs of sources with both `bidstream_enabled` and `mirror_enabled` (off by default), leaving out UIDs whose owning provider's TCF vendor consent is denied. Never trusted by the server over KV. Filled deterministically up to its size limit. | +| D6b | Mirror consumer | The tsjs change that reads the mirror and hands IDs to Prebid is a separate optional PR. | +| D7 | Expiry | Declared by the writing provider per UID. A UID without `exp` is served until replaced or removed. Browser and batch IDs carry no `exp` and no owner. | +| D8 | Conflicts | On a provider-owned source, the provider's UIDs replace that source's UIDs. Browser ingest fills only missing sources and never overwrites provider-owned UIDs. Batch sync rejects provider-owned sources. | +| D9 | Removal | Only consent withdrawal (the existing whole-row tombstone) is in scope. A tombstone wins over concurrent and later collection; IDs return only after consent is granted again. | +| D10 | Pull sync | Re-expressed as the first provider on the trait; the separate post-send dispatcher is removed. | +| D11 | Consent | Existing EID-persistence consent check, plus an optional per-provider TCF vendor ID that must also have vendor consent. | +| D12 | KV schema | Stay on `v: 1` with additive optional fields; no migration. | +| D13 | Other consumers | Provider IDs appear in `x-ts-eids` and `/_ts/api/v1/identify`, subject to `bidstream_enabled`, expiry and consent. | §5.2.1 (opt-in `collect = "after_response"`) and §5.6 (refresh state) extend D2 and D3 additively; defaults are unchanged. @@ -1057,10 +1057,6 @@ pub trait IdentityCoordinator { transport: &T, pending: PendingLookups, ) -> LookupResults; - - /// Builds the mirror cookie from the row as it stands after this - /// navigation: the committed row, or the snapshot when no write landed. - fn mirror(&self, row: Option<&KvEntry>, consent: &ConsentContext, now: UnixSeconds) -> MirrorCookie; } ``` @@ -1232,7 +1228,9 @@ where Err(report) => ec.record_write_failure(&report), } } - ec.apply_cookies(&mut response, row.as_ref(), coordinator.mirror(row.as_ref(), ec.consent(), ec.now())); + // `ec_finalize_response` then applies the mirror (§10) with + // `ec::mirror::build_mirror(row, consent, partners, now)`. + ec.apply_cookies(&mut response, row.as_ref()); response } ``` @@ -1422,18 +1420,27 @@ still due. | Value | base64url of `{"v":1,"eids":[...]}`; each entry is an OpenRTB EID whose UIDs may carry `exp` | | Size | Encoded value at most 3,584 bytes | -- **Contents:** live, provider-owned UIDs of `bidstream_enabled` sources, with - their EID-level and UID-level fields. +- **Contents:** live, provider-owned UIDs of sources whose partner sets both + `bidstream_enabled` and `mirror_enabled` (off by default), with their + EID-level and UID-level fields. A UID whose owning provider has a + `tcf_vendor_id` is left out while the request denies that vendor consent. - **Fill order:** sources in `[[ec.partners]]` config order; entries and UIDs in stored order. Each UID is added if the encoded value still fits; otherwise it is skipped and filling continues. KV keeps the complete set. -- **Set** on every consented navigation response from the row as it stands - after the navigation: the committed row, or the snapshot when no write - landed. Setting a cookie is not a KV write, and this restores a mirror the - browser lost. -- **Expired** (`Max-Age=0`) on any response where EID consent is absent or - withdrawn, whatever happened to the KV write, and on a navigation whose - mirror would be empty. +- **Built** by `ec::mirror::build_mirror(row, consent, partners, now)`, called + from `ec_finalize_response`, on consented navigations from the row as it + stands after the navigation: the committed row, or the snapshot when no + write landed. It is **set** only when its value differs from the one the + request carried, so unchanged responses carry no `Set-Cookie`; a mirror the + browser lost is restored on the next navigation. +- **Expired** (`Max-Age=0`) only when the request carried it: on any response + where EID consent is absent or withdrawn, whatever happened to the KV write; + on a navigation whose mirror would be empty; and, where the mirror cannot be + rebuilt (non-navigations, or a navigation whose row cannot be read), when the + request denies TCF vendor consent to the owning provider of a mirrored + partner. +- **Left untouched** on a navigation whose row cannot be read authoritatively + (KV failure), unless the vendor rule above expires it. - **The server never reads it.** Readers on the page must drop UIDs whose `exp` has passed and strip `exp` before use. @@ -1583,15 +1590,15 @@ PR A alone keeps the old post-send pull-sync write as a separate channel adds to it. PR B removes it. Within each pull request the units are separate, ordered commits so they stay reviewable. -| PR | Title | Contents | Depends on | -| --- | ---------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------- | -| PR0 | Spec | This document. | — | -| PR1 | Full-fidelity EID storage and resolution | `ec::eid_types` value types (§6.2); `KvEid`/`KvUid` additive schema; EID-level fields on `crate::openrtb::Eid`; resolver drops UIDs that are not live; auction merge (§11.1); `x-ts-eids` and identify use it. No new writers. | PR0 | -| PR2 | Page-load write permit | Navigation classifier; `PageLoadWrite` and `commit_page_load`; single attempt; non-navigation requests read-only except withdrawal tombstones; browser ingest on navigation only; withdrawal handling (§3.1, §5.7); integration-proxy EC minting stops. | PR0 | -| PR3 | Provider trait and coordinator | `ec::provider` (`IdentityProvider`, `ProviderKind`), `ec::coordinator` (`IdentityCoordinator`, `ProviderCoordinator

`, `LookupTransport`, `PlatformLookupTransport`), provider crate with no kinds, config and validation, consent, allowlist, budgeted fan-out, validation, merge into the navigation write, provider-owned skip in ingest, metrics, example config. | PR1, PR2, #1433 | -| PR4 | Batch sync on provider-owned sources | Reject and count records for sources an enabled provider claims. | PR3 | -| PR5 | Pull sync as a provider | `pull-sync` kind built from existing `[[ec.partners]]` pull-sync settings with no config change; existing per-partner rate limit kept as provider-scoped eligibility; post-send dispatcher and completeness marker removed, leftover marker cookies expired. Behaviour change for existing pull-sync partners: their sources become provider-owned, so browser ingest skips them and, with PR4, batch sync rejects them; once PR6 lands their IDs are mirrored when `bidstream_enabled`. | PR3 | -| PR6 | Mirror cookie | Adds `IdentityCoordinator::mirror` (PR3 ships the trait without it); encoding, fill order, set and expiry rules (§10). | PR3 | +| PR | Title | Contents | Depends on | +| --- | ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------- | +| PR0 | Spec | This document. | — | +| PR1 | Full-fidelity EID storage and resolution | `ec::eid_types` value types (§6.2); `KvEid`/`KvUid` additive schema; EID-level fields on `crate::openrtb::Eid`; resolver drops UIDs that are not live; auction merge (§11.1); `x-ts-eids` and identify use it. No new writers. | PR0 | +| PR2 | Page-load write permit | Navigation classifier; `PageLoadWrite` and `commit_page_load`; single attempt; non-navigation requests read-only except withdrawal tombstones; browser ingest on navigation only; withdrawal handling (§3.1, §5.7); integration-proxy EC minting stops. | PR0 | +| PR3 | Provider trait and coordinator | `ec::provider` (`IdentityProvider`, `ProviderKind`), `ec::coordinator` (`IdentityCoordinator`, `ProviderCoordinator

`, `LookupTransport`, `PlatformLookupTransport`), provider crate with no kinds, config and validation, consent, allowlist, budgeted fan-out, validation, merge into the navigation write, provider-owned skip in ingest, metrics, example config. | PR1, PR2, #1433 | +| PR4 | Batch sync on provider-owned sources | Reject and count records for sources an enabled provider claims. | PR3 | +| PR5 | Pull sync as a provider | `pull-sync` kind built from existing `[[ec.partners]]` pull-sync settings with no config change; existing per-partner rate limit kept as provider-scoped eligibility; post-send dispatcher and completeness marker removed, leftover marker cookies expired. Behaviour for existing pull-sync partners: browser ingest and batch sync keep filling their sources, as before; pull sync fills a source only when it holds no UID. Pull-sync UIDs are provider-written, so once PR6 lands they are mirrored when the partner sets `bidstream_enabled` and `mirror_enabled`. | PR3 | +| PR6 | Mirror cookie | `ec::mirror::build_mirror`, called from `ec_finalize_response`; per-partner `mirror_enabled` opt-in; encoding, fill order, set and expiry rules (§10). | PR3 | ### 13.2 Optional and future (Tech Lead decision) @@ -1660,9 +1667,9 @@ request the units are separate, ordered commits so they stay reviewable. 20. **Redirect.** The navigation's origin response is a 302. → No KV write; the redirect target navigation may write. 21. **Pull-sync partner after PR5.** A partner with pull sync enabled also - receives a browser `ts-eids` UID and a batch record. → Both are ignored or - rejected as for any provider-owned source; the pull-sync provider's UID is - stored and served. + receives a browser `ts-eids` UID and a batch record. → Both still fill the + source when it holds no UID, as before PR5; pull sync is not called for a + source that already holds a live UID, and fills it only when it holds none. 22. **After-response provider.** Provider A is `collect = "page"` and answers in 50 ms; provider B is `collect = "after_response"` and answers in 900 ms; page budget 200 ms. → The response is sent without waiting for B; From 7cb536992f9e844e53120c212c4211ac8646cca2 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 03:39:51 +0530 Subject: [PATCH 16/20] Adapt pull sync tests to the base's snapshot and test-support changes Coordinator tests that dispatch a lookup now use a live row, since a missing row skips every provider; restore the WriteFailingEcKv test import. Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/coordinator/mod.rs | 8 ++++---- crates/trusted-server-core/src/ec/kv.rs | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 7c47df6de2..b352001d1f 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -1571,7 +1571,7 @@ mod tests { let coordinator = pull_sync_pair(NonZeroUsize::new(1)); let transport = CannedTransport::new(not_found_reply); - let results = run(&coordinator, &transport, &missing(), &granting()); + let results = run(&coordinator, &transport, &live_row(), &granting()); assert_eq!(transport.sent.borrow().len(), 1, "should send one lookup"); let statuses: Vec = results.iter().map(|lookup| lookup.status).collect(); @@ -1654,7 +1654,7 @@ mod tests { }) }); - let results = run(&coordinator, &transport, &missing(), &granting()); + let results = run(&coordinator, &transport, &live_row(), &granting()); let lookup = results.iter().next().expect("should report the provider"); assert_eq!( @@ -1670,7 +1670,7 @@ mod tests { let coordinator = pull_sync_pair(None); let transport = CannedTransport::new(not_found_reply); - run(&coordinator, &transport, &missing(), &granting()); + run(&coordinator, &transport, &live_row(), &granting()); assert_eq!(transport.sent.borrow().len(), 2, "should send both lookups"); } @@ -2000,7 +2000,7 @@ mod tests { let results = run_with( &coordinator, &transport, - &missing(), + &live_row(), &granting(), &FailingLimiter, ); diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs index fb78da178f..bd768b0615 100644 --- a/crates/trusted-server-core/src/ec/kv.rs +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -1319,7 +1319,7 @@ mod tests { use super::*; use crate::ec::checked_current_timestamp; use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, UidValue, UnixSeconds}; - use crate::ec::kv_backend::test_support::InMemoryEcKv; + use crate::ec::kv_backend::test_support::{InMemoryEcKv, WriteFailingEcKv}; use crate::ec::kv_types::{KvEid, KvUid}; fn snapshot_ec_id() -> String { From 8a8e9198b9427695b168598b904a44c579decdb9 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 14:37:01 +0530 Subject: [PATCH 17/20] Expire the mirror only when it holds a denied vendor's UIDs A carried mirror was expired whenever any mirrored partner's owning vendor was denied, so a visitor who denied one of two mirrored vendors had the mirror set on each navigation and expired on the next non-navigation. Expire it only when it holds a source whose owning vendor is denied, or when its value does not decode. Drop the stale pull-sync clause from the provider CHANGELOG entry and align the guide and spec wording. Signed-off-by: dhruv8sh --- CHANGELOG.md | 2 +- crates/trusted-server-core/src/ec/finalize.rs | 169 ++++++++++++++++-- crates/trusted-server-core/src/ec/mirror.rs | 21 ++- docs/guide/edge-cookies.md | 2 +- .../2026-10-30-s2s-eid-collection-design.md | 10 +- 5 files changed, 180 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 704f72d79c..86f122b3a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 On Fastly the final snapshot may wait up to 1 ms per partially received body so the host can surface bytes that already arrived, and reads on into the body of a response whose headers arrived by the cutoff instead of abandoning it. -- Added server-side identity providers (`[[ec.identity_providers]]`, Fastly only). Enabled providers look up a visitor's IDs on publisher navigations within `ec.identity_budget_ms` (1–1000, default 200), or after the response within `ec.identity_after_response_budget_ms` (1–5000, default 2000) for `collect = "after_response"`. A provider owns its `sources`: batch sync rejects them with the new `provider_owned` reason, pull sync skips them, and browser IDs never fill them. Set `tcf_vendor_id` to require TCF vendor consent for lookups and for every ID of the provider's sources in bidstream, `x-ts-eids` and `/identify`, browser-sent IDs included; a disabled provider keeps that gate. See [Identity providers](docs/guide/configuration.md#identity-providers). +- Added server-side identity providers (`[[ec.identity_providers]]`, Fastly only). Enabled providers look up a visitor's IDs on publisher navigations within `ec.identity_budget_ms` (1–1000, default 200), or after the response within `ec.identity_after_response_budget_ms` (1–5000, default 2000) for `collect = "after_response"`. A provider owns its `sources`: batch sync rejects them with the new `provider_owned` reason, and browser IDs never fill them. Set `tcf_vendor_id` to require TCF vendor consent for lookups and for every ID of the provider's sources in bidstream, `x-ts-eids` and `/identify`, browser-sent IDs included; a disabled provider keeps that gate. See [Identity providers](docs/guide/configuration.md#identity-providers). - Pull sync now runs as a `pull-sync` identity provider built from the existing `[[ec.partners]]` settings, after the response on top-level publisher navigations from recognized browsers, within `ec.identity_after_response_budget_ms` shared with other after-response providers. It no longer runs after `/auction` responses, on prefetches or on navigations without Fetch Metadata, and a partner slower than the budget is not filled, where each pull used to wait up to 15 seconds. `ec.pull_sync_concurrency` now caps the partners pulled per navigation (`0` means no limit; it used to mean one at a time), partner order rotates hourly, and a partner past the cap logs `skipped_concurrency_cap`. The `ts-ec-pull-complete` marker is retired and leftover cookies are expired; without it, a returning navigation reads the EC row before the origin response even when nothing else needs it. `kind = "pull-sync"` in `[[ec.identity_providers]]` is rejected, as is a configured provider that claims a pull-sync partner's source. - Added `mirror_enabled` on `[[ec.partners]]` (default `false`; requires `bidstream_enabled`). The partner's provider-written UIDs are mirrored to the JavaScript-readable `__Host-ts-eids-mirror` cookie, leaving out UIDs whose owning provider lacks TCF vendor consent. See [EID Mirror Cookie](docs/guide/edge-cookies.md#eid-mirror-cookie). - **Breaking:** For a partner with pull sync enabled, a resolved `ts_pull_token` that cannot form an `Authorization: Bearer` header, such as one ending in a newline, now fails config validation and stops startup. Before, only that partner's pull sync failed, after the page was sent. Check pull-token secrets before deploying. diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index d287462d62..1b3819cd95 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -12,7 +12,7 @@ use edgezero_core::body::Body as EdgeBody; use http::Response; use super::consent::{ec_consent_granted, ec_consent_withdrawn}; -use crate::consent::{ConsentContext, allows_eid_persistence, tcf_vendor_consent_denied}; +use crate::consent::{ConsentContext, allows_eid_persistence}; use crate::constants::{COOKIE_TS_EC_PULL_COMPLETE, COOKIE_TS_EIDS_MIRROR}; use crate::settings::Settings; @@ -22,10 +22,10 @@ use super::eid_types::UnixSeconds; use super::generation::{generate_ec_id, is_valid_ec_id}; use super::kv::{KvIdentityGraph, PartnerIdUpdate}; use super::kv_types::KvEntry; -use super::mirror::{MirrorCookie, build_mirror, mirror_consent}; +use super::mirror::{MirrorCookie, build_mirror, mirror_consent, mirror_sources}; use super::page_load::{PageLoadWrite, RowChange}; use super::prebid_eids::collect_eid_updates; -use super::registry::PartnerRegistry; +use super::registry::{PartnerConfig, PartnerRegistry}; use super::write_plan::{WritePlanInput, plan_write}; use super::{EcKvSnapshot, current_timestamp, log_id}; @@ -138,18 +138,29 @@ fn apply_mirror( } } -/// Whether the request carries a mirror and denies TCF vendor consent to the -/// owning provider of a mirrored partner, so the mirror may hold UIDs that -/// must no longer reach the page. +/// Whether the request carries a mirror that holds UIDs of a mirrored partner +/// whose owning provider's TCF vendor is denied, so those UIDs must no longer +/// reach the page. +/// +/// A carried value that does not decode as a mirror may hold anything, so it +/// counts as holding denied UIDs whenever any mirrored vendor is denied. fn mirror_vendor_denied(ec_context: &EcContext, registry: &PartnerRegistry) -> bool { - ec_context.mirror_cookie_present() - && registry.in_config_order().any(|partner| { - partner.bidstream_enabled - && partner.mirror_enabled - && partner - .owner_tcf_vendor_id - .is_some_and(|vendor| tcf_vendor_consent_denied(ec_context.consent(), vendor)) - }) + let Some(value) = ec_context.mirror_cookie() else { + return false; + }; + let denied = |partner: &PartnerConfig| { + partner.bidstream_enabled + && partner.mirror_enabled + && partner.owner_vendor_consent_denied(ec_context.consent()) + }; + if !registry.in_config_order().any(denied) { + return false; + } + mirror_sources(value).is_none_or(|sources| { + sources + .iter() + .any(|source| registry.find_by_source_domain(source).is_some_and(denied)) + }) } /// The row the mirror is built from. @@ -2178,6 +2189,136 @@ mod tests { ); } + /// Runs a mirror request with two mirrored partners: `a.example.com`, + /// whose owning vendor 1111 is granted, and `b.example.com`, whose owning + /// vendor 9999 is denied. The lookup finds UIDs for both. + fn run_split_vendor_mirror_request( + mirror: Option<&str>, + navigation: bool, + ) -> Response { + let settings = create_test_settings(); + let ec_id = sample_ec_id("mirr01"); + let graph = KvIdentityGraph::in_memory("mirror_store"); + let live = KvEntry::new( + &granting_consent(), + None, + current_timestamp(), + &settings.publisher.domain, + ); + graph + .create(&ec_id, &live) + .expect("should seed the live row"); + let mut ec_context = + returning_user_context(&ec_id, graph.load_snapshot(&ec_id), navigation); + *ec_context.consent_mut() = consent_granting_vendors(vec![1111]); + ec_context.set_mirror_cookie_for_test(mirror); + let entry = |source: &str| { + let uid = KvUid::new( + UidValue::try_from("provider-uid").expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::Never, + ); + ( + SourceDomain::try_from(source).expect("should build source"), + vec![KvEid::new(EidFields::default(), vec![uid])], + ) + }; + ec_context.set_lookup_results(LookupResults::new( + &ec_id, + vec![ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Found { + kept: 2, + dropped: DropCounts::default(), + }, + entries: vec![entry("a.example.com"), entry("b.example.com")], + cleared: Vec::new(), + }], + )); + let partners = ["a.example.com", "b.example.com"].map(|source| { + let mut partner = make_partner(source); + partner.mirror_enabled = true; + partner + }); + let mut registry = PartnerRegistry::from_config(&partners).expect("should build registry"); + registry.mark_provider_owned("a.example.com", NonZeroU16::new(1111)); + registry.mark_provider_owned("b.example.com", NonZeroU16::new(9999)); + let mut response = empty_response(); + + ec_finalize_response( + &settings, + &mut ec_context, + Some(&graph), + ®istry, + None, + None, + &mut response, + ); + response + } + + fn mirror_value(cookie: &str) -> &str { + cookie + .strip_prefix("__Host-ts-eids-mirror=") + .and_then(|rest| rest.split(';').next()) + .expect("should carry a mirror value") + } + + #[test] + fn partial_vendor_denial_keeps_a_mirror_without_the_denied_vendor() { + // Arrange + let navigation = run_split_vendor_mirror_request(None, true); + let set = mirror_cookies(&navigation); + assert_eq!(set.len(), 1, "should set the mirror on navigation"); + let value = mirror_value(&set[0]).to_owned(); + let sources = mirror_sources(&value).expect("should decode the mirror"); + assert_eq!( + sources, + vec!["a.example.com".to_owned()], + "should mirror only the granted vendor's partner" + ); + + // Act + let non_navigation = run_split_vendor_mirror_request(Some(&value), false); + let next_navigation = run_split_vendor_mirror_request(Some(&value), true); + + // Assert + assert!( + mirror_cookies(&non_navigation).is_empty(), + "should keep a mirror that holds no denied vendor's UIDs" + ); + assert!( + mirror_cookies(&next_navigation).is_empty(), + "should not re-set an unchanged mirror on the next navigation" + ); + } + + #[test] + fn partial_vendor_denial_expires_a_mirror_holding_the_denied_vendor() { + // Arrange + let json = serde_json::json!({ + "v": 1, + "eids": [ + {"source": "a.example.com", "uids": [{"id": "provider-uid", "atype": 1}]}, + {"source": "b.example.com", "uids": [{"id": "provider-uid", "atype": 1}]}, + ], + }); + let value = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(json.to_string()); + + // Act + let response = run_split_vendor_mirror_request(Some(&value), false); + + // Assert + let cookies = mirror_cookies(&response); + assert_eq!(cookies.len(), 1, "should emit one mirror cookie"); + assert!( + cookies[0].contains("Max-Age=0"), + "should expire a mirror holding a denied vendor's UIDs" + ); + } + #[test] fn vendor_withdrawal_without_a_mirror_cookie_emits_nothing() { let response = run_mirror_request_with_owner( diff --git a/crates/trusted-server-core/src/ec/mirror.rs b/crates/trusted-server-core/src/ec/mirror.rs index be7a933239..36223d569d 100644 --- a/crates/trusted-server-core/src/ec/mirror.rs +++ b/crates/trusted-server-core/src/ec/mirror.rs @@ -4,7 +4,7 @@ use base64::Engine as _; use serde_json::{Map, Value}; use crate::consent::ConsentContext; -use crate::consent::{allows_eid_persistence, tcf_vendor_consent_denied}; +use crate::consent::allows_eid_persistence; use crate::ec::consent::ec_consent_granted; use crate::ec::eid_types::{Expiry, UnixSeconds}; use crate::ec::kv_types::{KvEid, KvEntry, KvUid}; @@ -67,9 +67,7 @@ pub fn build_mirror( for partner in partners.in_config_order().filter(|partner| { partner.bidstream_enabled && partner.mirror_enabled - && !partner - .owner_tcf_vendor_id - .is_some_and(|vendor| tcf_vendor_consent_denied(consent, vendor)) + && !partner.owner_vendor_consent_denied(consent) }) { let Some(record) = row.ids.get(&partner.source_domain) else { continue; @@ -194,6 +192,21 @@ fn open_uids_array(mut header: String) -> Option { }) } +/// The `source` of each EID in an encoded mirror value, or `None` when the +/// value does not decode as a mirror. +pub(crate) fn mirror_sources(value: &str) -> Option> { + let bytes = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(value) + .ok()?; + let mirror: Value = serde_json::from_slice(&bytes).ok()?; + mirror + .get("eids")? + .as_array()? + .iter() + .map(|eid| eid.get("source")?.as_str().map(str::to_owned)) + .collect() +} + /// Whether consent allows the mirror: EC consent and EID persistence. #[must_use] pub(crate) fn mirror_consent(consent: &ConsentContext) -> bool { diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index 09613f1c7b..8dd9382558 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -281,7 +281,7 @@ Trusted Server can mirror provider-written UIDs into a JavaScript-readable cooki - **Contents source**: live UIDs written by an identity provider. Ownerless UIDs, from browser ingest, batch sync or legacy pull sync, are never mirrored. - **Pull-sync UIDs**: pull sync writes provider-owned UIDs, so a pull-sync partner with `mirror_enabled = true` has its UIDs mirrored. Any script on the page can read them, and the cookie is sent on every same-origin request, including to the publisher origin. Leave `mirror_enabled` off for a partner whose UIDs must not reach the page; its UIDs still reach the bidstream when `bidstream_enabled` is set. - **Vendor consent**: a UID whose owning provider has a `tcf_vendor_id` is left out while the request's TCF string denies that vendor, as it is from the bidstream. -- **Updates** (best effort): page navigations rebuild it from the row after that navigation's write. It is expired only when the request carried it, either because no live provider UID qualifies or because EC and EID consent is missing (on any response). Where it cannot be rebuilt (other responses, or a navigation whose row cannot be read), it is also expired when the request denies TCF vendor consent to the owning provider of a mirrored partner. Responses to browsers without a mirror get no mirror `Set-Cookie`. When the identity row cannot be read (KV failure) the mirror is left untouched, and a mirror that cannot be built is skipped without failing the response. +- **Updates** (best effort): page navigations rebuild it from the row after that navigation's write. It is expired only when the request carried it, either because no live provider UID qualifies or because EC and EID consent is missing (on any response). Where it cannot be rebuilt (other responses, or a navigation whose row cannot be read), it is also expired when it holds UIDs of a mirrored partner whose owning provider's TCF vendor the request denies, or when its value does not decode while any such vendor is denied. Responses to browsers without a mirror get no mirror `Set-Cookie`. When the identity row cannot be read (KV failure) the mirror is left untouched, and a mirror that cannot be built is skipped without failing the response. Readers must drop UIDs whose `exp` (Unix seconds) has passed and strip `exp` before using a UID. diff --git a/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md b/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md index d9b1259c00..f60e1d23e4 100644 --- a/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md +++ b/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md @@ -1436,12 +1436,14 @@ still due. - **Expired** (`Max-Age=0`) only when the request carried it: on any response where EID consent is absent or withdrawn, whatever happened to the KV write; on a navigation whose mirror would be empty; and, where the mirror cannot be - rebuilt (non-navigations, or a navigation whose row cannot be read), when the - request denies TCF vendor consent to the owning provider of a mirrored - partner. + rebuilt (non-navigations, or a navigation whose row cannot be read), when it + holds UIDs of a mirrored partner whose owning provider's TCF vendor the + request denies. A carried value that does not decode is treated as holding + them whenever any mirrored partner's vendor is denied. - **Left untouched** on a navigation whose row cannot be read authoritatively (KV failure), unless the vendor rule above expires it. -- **The server never reads it.** Readers on the page must drop UIDs whose `exp` +- **The server reads it only to decide whether to set or expire it**, and + never trusts it over KV. Readers on the page must drop UIDs whose `exp` has passed and strip `exp` before use. ## 11. Consumers From 58b34c40b521bc50b6a4f1d4e7e8bfdb79b79334 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 17:34:42 +0530 Subject: [PATCH 18/20] Adapt pull sync to the vendor providers' optional endpoint and lookup inputs The vendors base makes a provider endpoint optional, names each provider in its build inputs, passes request cookies and the clock to lookups and records opt-out memory in lookup results. Require an endpoint for pull sync and update the pull-sync tests and fixtures to match. Signed-off-by: dhruv8sh --- .../trusted-server-core/src/ec/after_response.rs | 1 + crates/trusted-server-core/src/ec/finalize.rs | 2 ++ crates/trusted-server-core/src/settings.rs | 2 +- .../src/pull_sync.rs | 14 +++++++++++--- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/crates/trusted-server-core/src/ec/after_response.rs b/crates/trusted-server-core/src/ec/after_response.rs index 58facd44f8..ada2c618cd 100644 --- a/crates/trusted-server-core/src/ec/after_response.rs +++ b/crates/trusted-server-core/src/ec/after_response.rs @@ -215,6 +215,7 @@ mod tests { status: LookupStatus::Skipped(SkipReason::NotDue), entries: Vec::new(), cleared: Vec::new(), + opt_out: OptOutMemory::Unchanged, }], ); diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index 1b3819cd95..b6553b650f 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -2045,6 +2045,7 @@ mod tests { vec![KvEid::new(EidFields::default(), vec![uid])], )], cleared: Vec::new(), + opt_out: OptOutMemory::Unchanged, }], ) } @@ -2235,6 +2236,7 @@ mod tests { }, entries: vec![entry("a.example.com"), entry("b.example.com")], cleared: Vec::new(), + opt_out: OptOutMemory::Unchanged, }], )); let partners = ["a.example.com", "b.example.com"].map(|source| { diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 30d8bd8959..34977b78a9 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -718,7 +718,7 @@ impl IdentityProviderConfig { id, kind, enabled: true, - endpoint, + endpoint: Some(endpoint), allowed_domains, sources, tcf_vendor_id: None, diff --git a/crates/trusted-server-identity-providers/src/pull_sync.rs b/crates/trusted-server-identity-providers/src/pull_sync.rs index 404dbfffc8..9895528359 100644 --- a/crates/trusted-server-identity-providers/src/pull_sync.rs +++ b/crates/trusted-server-identity-providers/src/pull_sync.rs @@ -62,9 +62,13 @@ impl ProviderKind for PullSync { Ok::<_, Report>(value) }) .transpose()?; + let endpoint = common.endpoint.cloned().ok_or_else(|| { + Report::new(ProviderConfigError::Invalid) + .attach("a pull-sync provider needs an endpoint") + })?; let one = NonZeroUsize::MIN; Ok(PullSyncProvider { - endpoint: common.endpoint.clone(), + endpoint, source: source.clone(), authorization, capabilities: ProviderCapabilities { @@ -136,7 +140,8 @@ impl IdentityProvider for PullSyncProvider { mod tests { use super::*; use trusted_server_core::consent::ConsentContext; - use trusted_server_core::ec::provider::{ClientSignals, OwnedEntries}; + use trusted_server_core::ec::eid_types::UnixSeconds; + use trusted_server_core::ec::provider::{ClientSignals, OwnedEntries, RequestCookies}; use trusted_server_core::redacted::Redacted; use trusted_server_core::settings::ProviderSecrets; @@ -148,7 +153,8 @@ mod tests { }; PullSync::build( &ProviderCommon { - endpoint: &endpoint, + id: "pull-sync:partner.example.com", + endpoint: Some(&endpoint), sources: &sources, secrets: &secrets, }, @@ -174,7 +180,9 @@ mod tests { publisher_domain: "www.example.com", due_sources: &due, current: OwnedEntries::empty(), + cookies: RequestCookies::default(), hashed_email: None, + now: UnixSeconds::new(1_700_000_000), }) .expect("should build request"); From 61efaf8c88a98a1eb0b06dc2c99d5b2f117e2306 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 17:38:11 +0530 Subject: [PATCH 19/20] Keep pull-sync UIDs within the legacy 512-byte limit and document the multi-vendor mirror gate Pull sync runs for existing deployments with no config change, but as a provider it stored UIDs up to 1024 bytes, which an older build rejects with the whole row on rollback. Ignore pull-sync UIDs over 512 bytes, as pull sync did before. The mirror is withheld while any TCF vendor gating its source is denied, including a disabled provider's. Test that case and align the docs, spec and changelog, which still named a single owning vendor. Signed-off-by: dhruv8sh --- CHANGELOG.md | 2 +- crates/trusted-server-core/src/config.rs | 2 +- crates/trusted-server-core/src/ec/finalize.rs | 5 +- crates/trusted-server-core/src/ec/mirror.rs | 61 +++++++++++++++++-- .../src/lib.rs | 3 +- .../src/pull_sync.rs | 41 +++++++++++++ docs/guide/edge-cookies.md | 4 +- .../2026-10-30-s2s-eid-collection-design.md | 42 +++++++------ 8 files changed, 129 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86f122b3a8..65ea4e6f93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 a response whose headers arrived by the cutoff instead of abandoning it. - Added server-side identity providers (`[[ec.identity_providers]]`, Fastly only). Enabled providers look up a visitor's IDs on publisher navigations within `ec.identity_budget_ms` (1–1000, default 200), or after the response within `ec.identity_after_response_budget_ms` (1–5000, default 2000) for `collect = "after_response"`. A provider owns its `sources`: batch sync rejects them with the new `provider_owned` reason, and browser IDs never fill them. Set `tcf_vendor_id` to require TCF vendor consent for lookups and for every ID of the provider's sources in bidstream, `x-ts-eids` and `/identify`, browser-sent IDs included; a disabled provider keeps that gate. See [Identity providers](docs/guide/configuration.md#identity-providers). - Pull sync now runs as a `pull-sync` identity provider built from the existing `[[ec.partners]]` settings, after the response on top-level publisher navigations from recognized browsers, within `ec.identity_after_response_budget_ms` shared with other after-response providers. It no longer runs after `/auction` responses, on prefetches or on navigations without Fetch Metadata, and a partner slower than the budget is not filled, where each pull used to wait up to 15 seconds. `ec.pull_sync_concurrency` now caps the partners pulled per navigation (`0` means no limit; it used to mean one at a time), partner order rotates hourly, and a partner past the cap logs `skipped_concurrency_cap`. The `ts-ec-pull-complete` marker is retired and leftover cookies are expired; without it, a returning navigation reads the EC row before the origin response even when nothing else needs it. `kind = "pull-sync"` in `[[ec.identity_providers]]` is rejected, as is a configured provider that claims a pull-sync partner's source. -- Added `mirror_enabled` on `[[ec.partners]]` (default `false`; requires `bidstream_enabled`). The partner's provider-written UIDs are mirrored to the JavaScript-readable `__Host-ts-eids-mirror` cookie, leaving out UIDs whose owning provider lacks TCF vendor consent. See [EID Mirror Cookie](docs/guide/edge-cookies.md#eid-mirror-cookie). +- Added `mirror_enabled` on `[[ec.partners]]` (default `false`; requires `bidstream_enabled`). The partner's provider-written UIDs are mirrored to the JavaScript-readable `__Host-ts-eids-mirror` cookie, leaving out a source's UIDs while any provider claiming it lacks TCF vendor consent. See [EID Mirror Cookie](docs/guide/edge-cookies.md#eid-mirror-cookie). - **Breaking:** For a partner with pull sync enabled, a resolved `ts_pull_token` that cannot form an `Authorization: Bearer` header, such as one ending in a newline, now fails config validation and stops startup. Before, only that partner's pull sync failed, after the page was sent. Check pull-token secrets before deploying. - Batch sync reports a new `row_full` item reason when a mapping would push the EC row past its new 16 KiB size limit, and continues with the remaining groups. - An EC row over 16 KiB written by an earlier release loses its largest partner records on its next write that adds an ID, so the added ID still lands. diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 504fc3b973..6971fae1e5 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -475,7 +475,7 @@ fn validate_identity_providers(ec: &Ec) -> Result<(), Report }) })?; // Pull-sync partners run as providers too; a source claimed by both would - // fail the coordinator build on every request. + // fail the coordinator build at startup. for config in ec.identity_providers.iter().filter(|config| config.enabled) { if let Some(source) = config.sources.iter().find(|source| { partners diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index b6553b650f..c43eb0431f 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -93,7 +93,7 @@ pub fn ec_finalize_response( /// /// Where the mirror cannot be rebuilt (non-navigations, or a navigation /// without a readable row), a carried mirror is expired when the request -/// denies TCF vendor consent to the owning provider of any mirrored partner. +/// denies TCF vendor consent to a vendor gating any mirrored partner it holds. fn apply_mirror( ec_context: &EcContext, registry: &PartnerRegistry, @@ -139,7 +139,8 @@ fn apply_mirror( } /// Whether the request carries a mirror that holds UIDs of a mirrored partner -/// whose owning provider's TCF vendor is denied, so those UIDs must no longer +/// with a denied gating TCF vendor (that of any configured provider claiming +/// its source, enabled or disabled), so those UIDs must no longer /// reach the page. /// /// A carried value that does not decode as a mirror may hold anything, so it diff --git a/crates/trusted-server-core/src/ec/mirror.rs b/crates/trusted-server-core/src/ec/mirror.rs index 36223d569d..a751e55864 100644 --- a/crates/trusted-server-core/src/ec/mirror.rs +++ b/crates/trusted-server-core/src/ec/mirror.rs @@ -37,9 +37,8 @@ impl MirrorValue { /// Builds the mirror from the row as it stands after this navigation. /// /// Holds live, provider-written UIDs of partners with both -/// `bidstream_enabled` and `mirror_enabled` whose owning provider's TCF vendor -/// consent is not denied, in -/// partner config order, as unpadded base64url JSON +/// `bidstream_enabled` and `mirror_enabled` while no TCF vendor gating the +/// source is denied consent, in partner config order, as unpadded base64url JSON /// `{"v":1,"eids":[...]}`. UIDs that would push the value past /// [`MAX_MIRROR_BYTES`] are skipped, as are UIDs that fail to serialize. /// Returns [`MirrorCookie::Expire`] when consent does not allow the EC or EID @@ -62,7 +61,8 @@ pub fn build_mirror( }; let mut mirror = MirrorBuilder::default(); // A partner opts in with `mirror_enabled`, and a UID never reaches the - // page while its owning provider's TCF vendor consent is denied, as it + // page while a TCF vendor gating its source (that of any configured + // provider claiming it, enabled or disabled) is denied consent, as it // never reaches the bidstream (`KvEntry::live_eids`). for partner in partners.in_config_order().filter(|partner| { partner.bidstream_enabled @@ -420,6 +420,59 @@ mod tests { ); } + #[test] + fn withholds_uids_while_a_disabled_providers_vendor_lacks_consent() { + // Arrange: enabled provider `new` (vendor 7) owns the source that + // disabled provider `old` (vendor 42) claimed, so both vendors gate it. + let entry = row(vec![uid("live", Expiry::Never)]); + let mut ec = crate::test_support::tests::create_test_settings().ec; + ec.partners = vec![ + serde_json::from_value(json!({ + "name": "ids", "source_domain": "ids.example.com", + "bidstream_enabled": true, "mirror_enabled": true + })) + .expect("should build partner"), + ]; + ec.identity_providers = vec![ + toml::from_str( + "id = \"new\"\nkind = \"k\"\ntcf_vendor_id = 7\nendpoint = \"https://ids.example.com/\"\nsources = [\"ids.example.com\"]", + ) + .expect("should parse enabled provider"), + toml::from_str( + "id = \"old\"\nkind = \"k\"\nenabled = false\ntcf_vendor_id = 42\nendpoint = \"https://ids.example.com/\"\nsources = [\"ids.example.com\"]", + ) + .expect("should parse disabled provider"), + ]; + let registry = PartnerRegistry::from_ec_with_kinds(&ec, |kind| kind == "k") + .expect("should build registry"); + let now = UnixSeconds::new(1000); + + // Act + let owner_only = build_mirror( + Some(&entry), + &consent_granting_vendors(vec![7]), + ®istry, + now, + ); + let both = build_mirror( + Some(&entry), + &consent_granting_vendors(vec![7, 42]), + ®istry, + now, + ); + + // Assert + assert_eq!( + owner_only, + MirrorCookie::Expire, + "should withhold the source while the disabled provider's vendor lacks consent" + ); + assert!( + matches!(both, MirrorCookie::Set(_)), + "should mirror the source once every gating vendor has consent" + ); + } + #[test] fn does_not_mirror_browser_ingested_uid_of_pull_sync_source() { let mut entry = KvEntry::tombstone(1000); diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index a937932b85..4ea3609844 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -81,7 +81,8 @@ fn excludes_where_gdpr_applies(kind: &str, source: &str) -> bool { } /// Builds the partner registry, marking the sources claimed by enabled -/// providers (including pull-sync partners) as provider-owned. +/// configured providers as provider-owned. A pull-sync partner's source is +/// not marked, so browser ingest and batch sync still fill it. /// /// The sources any configured provider, enabled or disabled, excludes where /// GDPR applies are marked too, so the UIDs already stored for them are diff --git a/crates/trusted-server-identity-providers/src/pull_sync.rs b/crates/trusted-server-identity-providers/src/pull_sync.rs index 9895528359..4b106988f2 100644 --- a/crates/trusted-server-identity-providers/src/pull_sync.rs +++ b/crates/trusted-server-identity-providers/src/pull_sync.rs @@ -7,6 +7,7 @@ use std::num::NonZeroUsize; use error_stack::Report; use serde::Deserialize; use trusted_server_core::ec::eid_types::SourceDomain; +use trusted_server_core::ec::kv_types::MAX_UID_LENGTH; use trusted_server_core::ec::provider::{ EidLevelFields, EidShape, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, ProviderConfigError, ProviderEid, ProviderError, ProviderExpiry, ProviderKind, @@ -119,6 +120,18 @@ impl IdentityProvider for PullSyncProvider { let Some(uid) = body.uid.filter(|uid| !uid.trim().is_empty()) else { return Ok(LookupOutcome::NoMatch); }; + // Pull sync ran before identity providers existed, so its UIDs keep + // the legacy 512-byte limit: an older build rejects a row holding a + // longer UID whole, which would break rollback for every deployment + // that already pulls. + if uid.len() > MAX_UID_LENGTH { + log::warn!( + "Pull sync: partner '{}' returned a UID over {MAX_UID_LENGTH} bytes ({}), ignoring it", + self.source.as_str(), + uid.len() + ); + return Ok(LookupOutcome::NoMatch); + } Ok(LookupOutcome::Found(vec![ProviderEid { source: Cow::Owned(self.source.as_str().to_owned()), inserter: None, @@ -239,4 +252,32 @@ mod tests { ); assert!(invalid.is_err(), "should reject invalid JSON"); } + + #[test] + fn ignores_a_uid_over_the_legacy_limit() { + let provider = provider(); + let headers = http::HeaderMap::new(); + let at_limit = serde_json::json!({ "uid": "a".repeat(MAX_UID_LENGTH) }).to_string(); + let over_limit = serde_json::json!({ "uid": "a".repeat(MAX_UID_LENGTH + 1) }).to_string(); + let respond = |body: &[u8]| { + provider + .parse_response(ProviderResponse { + status: http::StatusCode::OK, + headers: &headers, + body, + context: None, + }) + .map(|outcome| matches!(outcome, LookupOutcome::Found(_))) + .expect("should parse") + }; + + assert!( + respond(at_limit.as_bytes()), + "should keep a UID at the legacy limit" + ); + assert!( + !respond(over_limit.as_bytes()), + "should ignore a UID an older build could not load" + ); + } } diff --git a/docs/guide/edge-cookies.md b/docs/guide/edge-cookies.md index 8dd9382558..2eef8c39e4 100644 --- a/docs/guide/edge-cookies.md +++ b/docs/guide/edge-cookies.md @@ -280,8 +280,8 @@ Trusted Server can mirror provider-written UIDs into a JavaScript-readable cooki - **Contents**: unpadded base64url JSON `{"v":1,"eids":[{"source":"ids.example.com","uids":[{"id":"...","atype":1,"exp":1767225600}]}]}`, partners in `[[ec.partners]]` order, at most 3584 bytes; UIDs that do not fit are left out - **Contents source**: live UIDs written by an identity provider. Ownerless UIDs, from browser ingest, batch sync or legacy pull sync, are never mirrored. - **Pull-sync UIDs**: pull sync writes provider-owned UIDs, so a pull-sync partner with `mirror_enabled = true` has its UIDs mirrored. Any script on the page can read them, and the cookie is sent on every same-origin request, including to the publisher origin. Leave `mirror_enabled` off for a partner whose UIDs must not reach the page; its UIDs still reach the bidstream when `bidstream_enabled` is set. -- **Vendor consent**: a UID whose owning provider has a `tcf_vendor_id` is left out while the request's TCF string denies that vendor, as it is from the bidstream. -- **Updates** (best effort): page navigations rebuild it from the row after that navigation's write. It is expired only when the request carried it, either because no live provider UID qualifies or because EC and EID consent is missing (on any response). Where it cannot be rebuilt (other responses, or a navigation whose row cannot be read), it is also expired when it holds UIDs of a mirrored partner whose owning provider's TCF vendor the request denies, or when its value does not decode while any such vendor is denied. Responses to browsers without a mirror get no mirror `Set-Cookie`. When the identity row cannot be read (KV failure) the mirror is left untouched, and a mirror that cannot be built is skipped without failing the response. +- **Vendor consent**: a source's UIDs are left out while the request's TCF string denies the `tcf_vendor_id` of any configured provider claiming that source, enabled or disabled, as they are from the bidstream. +- **Updates** (best effort): page navigations rebuild it from the row after that navigation's write. It is expired only when the request carried it, either because no live provider UID qualifies or because EC and EID consent is missing (on any response). Where it cannot be rebuilt (other responses, or a navigation whose row cannot be read), it is also expired when it holds UIDs of a mirrored partner with such a denied vendor, or when its value does not decode while any such vendor is denied. Responses to browsers without a mirror get no mirror `Set-Cookie`. When the identity row cannot be read (KV failure) the mirror is left untouched, and a mirror that cannot be built is skipped without failing the response. Readers must drop UIDs whose `exp` (Unix seconds) has passed and strip `exp` before using a UID. diff --git a/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md b/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md index f60e1d23e4..e1babc0e97 100644 --- a/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md +++ b/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md @@ -72,22 +72,22 @@ decision. These were agreed before drafting and are binding on the rest of the spec. -| # | Topic | Decision | -| --- | ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| D1 | Which request may write | Only the top-level document navigation. `/auction`, `/_ts/page-bids` and integration-proxy requests never write KV. | -| D2 | Provider timing | Provider requests start when the navigation starts, run concurrently with the origin fetch, and are collected within a shared total budget of about 200 ms. Their results join the navigation's write. | -| D3 | Which navigations look up | Any navigation where an enabled provider has a due source (missing or expired UID). No-match results are not cached in V1. | -| D4 | Failed write | Not retried in that page load. KV and the mirror cookie stay unchanged. A new EC is not issued if its row was not created. The next navigation is a new page load and may write. | -| D5 | Availability | Provider IDs are guaranteed to `/auction` from the next page load. On the same page they are used if the write already completed. | -| D6 | Mirror cookie | JavaScript-readable. Holds provider-owned UIDs of sources with both `bidstream_enabled` and `mirror_enabled` (off by default), leaving out UIDs whose owning provider's TCF vendor consent is denied. Never trusted by the server over KV. Filled deterministically up to its size limit. | -| D6b | Mirror consumer | The tsjs change that reads the mirror and hands IDs to Prebid is a separate optional PR. | -| D7 | Expiry | Declared by the writing provider per UID. A UID without `exp` is served until replaced or removed. Browser and batch IDs carry no `exp` and no owner. | -| D8 | Conflicts | On a provider-owned source, the provider's UIDs replace that source's UIDs. Browser ingest fills only missing sources and never overwrites provider-owned UIDs. Batch sync rejects provider-owned sources. | -| D9 | Removal | Only consent withdrawal (the existing whole-row tombstone) is in scope. A tombstone wins over concurrent and later collection; IDs return only after consent is granted again. | -| D10 | Pull sync | Re-expressed as the first provider on the trait; the separate post-send dispatcher is removed. | -| D11 | Consent | Existing EID-persistence consent check, plus an optional per-provider TCF vendor ID that must also have vendor consent. | -| D12 | KV schema | Stay on `v: 1` with additive optional fields; no migration. | -| D13 | Other consumers | Provider IDs appear in `x-ts-eids` and `/_ts/api/v1/identify`, subject to `bidstream_enabled`, expiry and consent. | +| # | Topic | Decision | +| --- | ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| D1 | Which request may write | Only the top-level document navigation. `/auction`, `/_ts/page-bids` and integration-proxy requests never write KV. | +| D2 | Provider timing | Provider requests start when the navigation starts, run concurrently with the origin fetch, and are collected within a shared total budget of about 200 ms. Their results join the navigation's write. | +| D3 | Which navigations look up | Any navigation where an enabled provider has a due source (missing or expired UID). No-match results are not cached in V1. | +| D4 | Failed write | Not retried in that page load. KV and the mirror cookie stay unchanged. A new EC is not issued if its row was not created. The next navigation is a new page load and may write. | +| D5 | Availability | Provider IDs are guaranteed to `/auction` from the next page load. On the same page they are used if the write already completed. | +| D6 | Mirror cookie | JavaScript-readable. Holds provider-owned UIDs of sources with both `bidstream_enabled` and `mirror_enabled` (off by default), leaving out UIDs of sources with a denied gating TCF vendor. Never trusted by the server over KV. Filled deterministically up to its size limit. | +| D6b | Mirror consumer | The tsjs change that reads the mirror and hands IDs to Prebid is a separate optional PR. | +| D7 | Expiry | Declared by the writing provider per UID. A UID without `exp` is served until replaced or removed. Browser and batch IDs carry no `exp` and no owner. | +| D8 | Conflicts | On a provider-owned source, the provider's UIDs replace that source's UIDs. Browser ingest fills only missing sources and never overwrites provider-owned UIDs. Batch sync rejects provider-owned sources. | +| D9 | Removal | Only consent withdrawal (the existing whole-row tombstone) is in scope. A tombstone wins over concurrent and later collection; IDs return only after consent is granted again. | +| D10 | Pull sync | Re-expressed as the first provider on the trait; the separate post-send dispatcher is removed. | +| D11 | Consent | Existing EID-persistence consent check, plus an optional per-provider TCF vendor ID that must also have vendor consent. | +| D12 | KV schema | Stay on `v: 1` with additive optional fields; no migration. | +| D13 | Other consumers | Provider IDs appear in `x-ts-eids` and `/_ts/api/v1/identify`, subject to `bidstream_enabled`, expiry and consent. | §5.2.1 (opt-in `collect = "after_response"`) and §5.6 (refresh state) extend D2 and D3 additively; defaults are unchanged. @@ -1422,8 +1422,10 @@ still due. - **Contents:** live, provider-owned UIDs of sources whose partner sets both `bidstream_enabled` and `mirror_enabled` (off by default), with their - EID-level and UID-level fields. A UID whose owning provider has a - `tcf_vendor_id` is left out while the request denies that vendor consent. + EID-level and UID-level fields. A source's UIDs are left out while the + request denies consent to any TCF vendor gating that source: the + `tcf_vendor_id` of every configured provider claiming it, enabled or + disabled (§8.6). - **Fill order:** sources in `[[ec.partners]]` config order; entries and UIDs in stored order. Each UID is added if the encoded value still fits; otherwise it is skipped and filling continues. KV keeps the complete set. @@ -1437,8 +1439,8 @@ still due. where EID consent is absent or withdrawn, whatever happened to the KV write; on a navigation whose mirror would be empty; and, where the mirror cannot be rebuilt (non-navigations, or a navigation whose row cannot be read), when it - holds UIDs of a mirrored partner whose owning provider's TCF vendor the - request denies. A carried value that does not decode is treated as holding + holds UIDs of a mirrored partner with a gating TCF vendor the request + denies. A carried value that does not decode is treated as holding them whenever any mirrored partner's vendor is denied. - **Left untouched** on a navigation whose row cannot be read authoritatively (KV failure), unless the vendor rule above expires it. From d3780ba13dd598836bb9992145d170a19bdb124b Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 17:38:46 +0530 Subject: [PATCH 20/20] Add the PR #1489 review rounds to the S2S EID spec Signed-off-by: dhruv8sh --- .../2026-10-30-s2s-eid-collection-design.md | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md b/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md index e1babc0e97..d4ef53416f 100644 --- a/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md +++ b/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md @@ -1830,3 +1830,40 @@ clock edges and privacy leaks. - Non-blocking, no change: a UID2 refresh that keeps failing is retried on every navigation until the token expires unless `rate_limit_per_hour` is set. The fix for overdue UID2 refresh state (834af2591) chose this bound. + +### Round 9 (2026-11-12, PR #1489 at 4b11d05a1, Claude Code (Opus 5.5) review agent) + +Angle: data flow end to end, from partner config and the registry through +pull sync dispatch, the after-response write and bidstream egress, to the +mirror cookie and its vendor-consent expiry, on the tree restacked onto PR +#1488. + +- Blocking, fixed (9638d7184): pull sync, now a provider, stored UIDs up to + 1024 bytes. It runs for existing deployments with no config change, so a + row holding a longer pull-sync UID broke rollback, which older builds + reject whole (§6.3, §12.3). Pull-sync UIDs over 512 bytes are ignored, as + legacy pull sync did. +- Restack: the mirror's vendor gate now uses the per-source vendor set from + PR #1487 round 5, so a disabled provider's denied vendor withholds the + source from the mirror and expires a carried mirror holding it, as in the + bidstream. Tested; §10, the guide and the changelog no longer name a single + owning vendor. +- Non-blocking, fixed: stale comments on pull-sync ownership and on when a + provider claiming a pull-sync source fails. + +### Round 10 (2026-11-12, PR #1489 at 9638d7184, Claude Code (Opus 5.5) review agent) + +Angle: adversarial. Malformed mirror cookies and pull-sync replies, config +combinations, adapter differences and privacy leaks. + +- No blocking findings. Re-checked forged, undecodable and oversized mirror + values (an undecodable one is expired whenever a mirrored vendor is denied; + one naming unknown sources is never trusted over KV); non-string, blank and + oversized pull-sync UIDs; zero rate limits and tokens that cannot form a + header (rejected at startup); `mirror_enabled` without + `bidstream_enabled`, and a configured provider or `kind = "pull-sync"` + claiming a pull-sync source (rejected); withdrawal and missing EID consent + expiring the mirror on every response; and the bot gate on pull sync. +- Non-blocking, no change: pull sync and provider-written mirror contents + exist only on Fastly; other adapters build the mirror from rows without + provider UIDs, so it stays unset there (§12.3 "On Fastly").