From bf0aeb2080c6f8b3fd928208c3b3185b1e9f0025 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 04:35:58 +0530 Subject: [PATCH 01/72] Add the ID5 identity provider Signed-off-by: dhruv8sh --- Cargo.lock | 6 + .../Cargo.toml | 6 + .../src/common.rs | 128 +++++ .../src/id5.rs | 446 ++++++++++++++++++ .../src/lib.rs | 24 +- .../plans/2026-11-02-identity-provider-id5.md | 38 ++ trusted-server.example.toml | 13 + 7 files changed, 651 insertions(+), 10 deletions(-) create mode 100644 crates/trusted-server-identity-providers/src/common.rs create mode 100644 crates/trusted-server-identity-providers/src/id5.rs create mode 100644 docs/superpowers/plans/2026-11-02-identity-provider-id5.md diff --git a/Cargo.lock b/Cargo.lock index d9cf35f9f..e243b3c3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6742,9 +6742,15 @@ dependencies = [ name = "trusted-server-identity-providers" version = "0.1.0" dependencies = [ + "bytes", "error-stack", + "http", + "log", + "serde", + "serde_json", "toml", "trusted-server-core", + "url", ] [[package]] diff --git a/crates/trusted-server-identity-providers/Cargo.toml b/crates/trusted-server-identity-providers/Cargo.toml index 1ff50dedf..aac7daab5 100644 --- a/crates/trusted-server-identity-providers/Cargo.toml +++ b/crates/trusted-server-identity-providers/Cargo.toml @@ -14,8 +14,14 @@ doctest = false workspace = true [dependencies] +bytes = { workspace = true } error-stack = { workspace = true } +http = { workspace = true } +log = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true, features = ["raw_value"] } trusted-server-core = { workspace = true } +url = { workspace = true } [dev-dependencies] toml = { workspace = true } diff --git a/crates/trusted-server-identity-providers/src/common.rs b/crates/trusted-server-identity-providers/src/common.rs new file mode 100644 index 000000000..ff7581d8d --- /dev/null +++ b/crates/trusted-server-identity-providers/src/common.rs @@ -0,0 +1,128 @@ +//! Helpers shared by the vendor provider modules. + +use std::borrow::Cow; +use std::net::IpAddr; + +use serde::Deserialize; +use serde_json::value::RawValue; +use trusted_server_core::ec::provider::{ProviderEid, ProviderExpiry, ProviderUid}; + +/// Returns whether `ip` is a public address worth sending to a vendor. +/// +/// Loopback, private, link-local, carrier-grade NAT and unique-local +/// addresses identify nothing outside the operator's network. +#[must_use] +pub(crate) fn is_routable_ip(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(v4) => { + let [first, second, ..] = v4.octets(); + let carrier_grade_nat = first == 100 && (second & 0xc0) == 64; + !(v4.is_loopback() + || v4.is_unspecified() + || v4.is_private() + || v4.is_link_local() + || v4.is_broadcast() + || carrier_grade_nat) + } + IpAddr::V6(v6) => { + if let Some(v4) = v6.to_ipv4_mapped() { + return is_routable_ip(IpAddr::V4(v4)); + } + let first = v6.segments()[0]; + let unique_local = (first & 0xfe00) == 0xfc00; + let link_local = (first & 0xffc0) == 0xfe80; + !(v6.is_loopback() || v6.is_unspecified() || unique_local || link_local) + } + } +} + +/// Joins GPP section IDs with commas, as vendors expect them. +#[must_use] +pub(crate) fn join_section_ids(ids: &[u16]) -> String { + ids.iter().map(u16::to_string).collect::>().join(",") +} + +/// A complete `OpenRTB` EID as a vendor returns it, borrowed from the body. +/// +/// Unknown fields are ignored so one new vendor field does not fail the +/// whole response. +#[derive(Debug, Deserialize)] +pub(crate) struct VendorEid<'body> { + #[serde(borrow)] + pub source: Cow<'body, str>, + #[serde(default, borrow)] + pub uids: Vec>, + #[serde(default, borrow)] + pub inserter: Option>, + #[serde(default, borrow)] + pub matcher: Option>, + #[serde(default)] + pub mm: Option, + #[serde(default, borrow)] + pub ext: Option<&'body RawValue>, +} + +/// One UID of a [`VendorEid`]. +#[derive(Debug, Deserialize)] +pub(crate) struct VendorUid<'body> { + #[serde(borrow)] + pub id: Cow<'body, str>, + #[serde(default)] + pub atype: Option, + #[serde(default, borrow)] + pub ext: Option<&'body RawValue>, +} + +impl<'body> VendorEid<'body> { + /// Converts to a [`ProviderEid`], giving every UID `expiry`. + pub(crate) fn into_provider_eid(self, expiry: ProviderExpiry) -> ProviderEid<'body> { + ProviderEid { + source: self.source, + inserter: self.inserter, + matcher: self.matcher, + mm: self.mm, + ext: self.ext, + uids: self + .uids + .into_iter() + .map(|uid| ProviderUid { + id: uid.id, + atype: uid.atype, + ext: uid.ext, + expiry, + }) + .collect(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn keeps_public_addresses_and_drops_local_ones() { + for ip in ["192.0.2.10", "198.51.100.7", "2001:db8::1"] { + assert!( + is_routable_ip(ip.parse().expect("should parse")), + "should keep {ip}" + ); + } + for ip in [ + "127.0.0.1", + "10.1.2.3", + "192.168.1.1", + "169.254.1.1", + "100.64.0.1", + "::1", + "fd00::1", + "fe80::1", + "::ffff:127.0.0.1", + ] { + assert!( + !is_routable_ip(ip.parse().expect("should parse")), + "should drop {ip}" + ); + } + } +} diff --git a/crates/trusted-server-identity-providers/src/id5.rs b/crates/trusted-server-identity-providers/src/id5.rs new file mode 100644 index 000000000..3115e9141 --- /dev/null +++ b/crates/trusted-server-identity-providers/src/id5.rs @@ -0,0 +1,446 @@ +//! ID5 identity provider: resolves the ID5 ID server side. +//! +//! Posts the request's consent and network signals to ID5's fetch endpoint. +//! It needs no login and sends no hashed email. ID5 returns complete EIDs +//! under `ids`, or the legacy `universal_uid`, which is given to the first +//! configured source. +//! +//! The request field names for the client IP and user agent follow ID5's +//! server-to-server integration and should be confirmed with ID5 for each +//! partner account. + +use std::borrow::Cow; +use std::collections::BTreeMap; +use std::num::NonZeroUsize; +use std::time::Duration; + +use bytes::Bytes; +use error_stack::{Report, ResultExt as _}; +use http::header::{CONTENT_TYPE, ORIGIN}; +use http::{HeaderMap, HeaderValue, Method}; +use serde::{Deserialize, Serialize}; +use trusted_server_core::ec::eid_types::SourceDomain; +use trusted_server_core::ec::provider::{ + EidLevelFields, EidShape, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, + ProviderCommon, ProviderConfigError, ProviderEid, ProviderError, ProviderExpiry, ProviderKind, + ProviderRequest, ProviderResponse, ProviderUid, SourceCapability, SourceInput, +}; +use url::Url; + +use crate::common::{VendorEid, is_routable_ip, join_section_ids}; + +/// The `universal_uid` ID5 returns for a visitor who opted out. +const OPTED_OUT_UID: &str = "0"; + +/// Most EIDs accepted from one ID5 response. +const MAX_EIDS: NonZeroUsize = NonZeroUsize::new(4).expect("should be non-zero"); + +/// Most UIDs accepted per ID5 EID. +const MAX_UIDS_PER_EID: NonZeroUsize = NonZeroUsize::new(2).expect("should be non-zero"); + +/// The ID5 provider kind. +pub struct Id5; + +/// Settings for an `id5` provider. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Id5Settings { + /// The ID5 partner number assigned to the publisher. + pub partner_id: u32, +} + +/// A built ID5 provider. +#[derive(Debug)] +pub struct Id5Provider { + capabilities: ProviderCapabilities, + url: Url, + partner_id: u32, +} + +impl ProviderKind for Id5 { + const KIND: &'static str = "id5"; + type Settings = Id5Settings; + type Provider = Id5Provider; + + fn build( + common: &ProviderCommon<'_>, + settings: Id5Settings, + ) -> Result> { + if common.sources.is_empty() { + return Err(Report::new(ProviderConfigError::Invalid) + .attach("an id5 provider needs at least one source")); + } + let mut url = common.endpoint.clone(); + url.path_segments_mut() + .map_err(|()| { + Report::new(ProviderConfigError::Invalid).attach("endpoint cannot hold a path") + })? + .pop_if_empty() + .push(&format!("{}.json", settings.partner_id)); + Ok(Id5Provider { + capabilities: ProviderCapabilities { + sources: common + .sources + .iter() + .map(|source| SourceCapability { + source: source.clone(), + input: SourceInput::Anonymous, + }) + .collect(), + shape: EidShape { + max_eids: MAX_EIDS, + max_uids_per_eid: MAX_UIDS_PER_EID, + eid_level_fields: EidLevelFields::Preserved, + }, + }, + url, + partner_id: settings.partner_id, + }) + } +} + +#[derive(Serialize)] +struct FetchBody<'a> { + partner: u32, + v: &'static str, + o: &'static str, + #[serde(rename = "nbPage")] + nb_page: u32, + gdpr: u8, + #[serde(skip_serializing_if = "Option::is_none")] + gdpr_consent: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + us_privacy: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + gpp_string: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + gpp_sid: Option, + tml: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + ua: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + ip: Option, +} + +#[derive(Deserialize)] +struct FetchReply<'body> { + #[serde(default, borrow)] + universal_uid: Option>, + #[serde(default, borrow)] + ids: BTreeMap, Id5Id<'body>>, + #[serde(default)] + cache_control: Option, +} + +#[derive(Deserialize)] +struct Id5Id<'body> { + #[serde(default, borrow)] + eid: Option>, +} + +#[derive(Deserialize)] +struct CacheControl { + max_age_sec: Option, +} + +impl Id5Provider { + fn primary_source(&self) -> &SourceDomain { + &self.capabilities.sources[0].source + } + + fn is_known(&self, source: &str) -> bool { + self.capabilities + .sources + .iter() + .any(|known| known.source.as_str() == source) + } +} + +impl IdentityProvider for Id5Provider { + fn capabilities(&self) -> &ProviderCapabilities { + &self.capabilities + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + let consent = lookup.consent; + let origin = format!("https://{}", lookup.publisher_domain); + let body = FetchBody { + partner: self.partner_id, + v: env!("CARGO_PKG_VERSION"), + o: "trusted-server", + nb_page: 1, + gdpr: u8::from(consent.gdpr_applies), + gdpr_consent: consent.raw_tc_string.as_deref(), + us_privacy: consent.raw_us_privacy.as_deref(), + gpp_string: consent.raw_gpp_string.as_deref(), + gpp_sid: consent.gpp_section_ids.as_deref().map(join_section_ids), + tml: &origin, + ua: lookup.client.user_agent, + ip: lookup + .client + .ip + .filter(|ip| is_routable_ip(*ip)) + .map(|ip| ip.to_string()), + }; + let body = serde_json::to_vec(&body).change_context(ProviderError::Request)?; + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + headers.insert( + ORIGIN, + HeaderValue::from_str(&origin).change_context(ProviderError::Request)?, + ); + Ok(ProviderRequest { + method: Method::POST, + url: self.url.clone(), + headers, + body: Bytes::from(body), + }) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + if !response.status.is_success() { + return Err(Report::new(ProviderError::InvalidResponse) + .attach(format!("ID5 returned status {}", response.status))); + } + let reply: FetchReply<'body> = + serde_json::from_slice(response.body).change_context(ProviderError::InvalidResponse)?; + let expiry = reply + .cache_control + .and_then(|cache| cache.max_age_sec) + .map_or(ProviderExpiry::Unspecified, |seconds| { + ProviderExpiry::After(Duration::from_secs(seconds)) + }); + + let mut eids: Vec> = reply + .ids + .into_values() + .filter_map(|id| id.eid) + .filter(|eid| self.is_known(&eid.source)) + .map(|eid| eid.into_provider_eid(expiry)) + .collect(); + let primary = self.primary_source().as_str(); + if !eids.iter().any(|eid| eid.source == primary) + && let Some(uid) = reply + .universal_uid + .filter(|uid| !uid.is_empty() && uid != OPTED_OUT_UID) + { + eids.push(ProviderEid { + source: Cow::Owned(primary.to_owned()), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids: vec![ProviderUid { + id: uid, + atype: Some(1), + ext: None, + expiry, + }], + }); + } + Ok(if eids.is_empty() { + LookupOutcome::NoMatch + } else { + LookupOutcome::Found(eids) + }) + } +} + +#[cfg(test)] +mod tests { + use http::StatusCode; + use serde_json::json; + use trusted_server_core::consent::types::ConsentContext; + use trusted_server_core::ec::provider::{ClientSignals, OwnedEntries, build_kind}; + use trusted_server_core::settings::{IdentityProviderConfig, ProviderSecrets}; + + use super::*; + + fn provider() -> Box { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "id5", + "kind": "id5", + "endpoint": "https://id5.example.com/g/v2", + "allowed_domains": ["id5.example.com"], + "sources": ["id5.example.com", "id5-link.example.com"], + "settings": { "partner_id": 1234 }, + })) + .expect("should parse provider config"); + build_kind::(&config, &ProviderSecrets::default()).expect("should build provider") + } + + fn respond<'body>( + provider: &dyn IdentityProvider, + headers: &'body HeaderMap, + body: &'body [u8], + ) -> LookupOutcome<'body> { + provider + .parse_response(ProviderResponse { + status: StatusCode::OK, + headers, + body, + }) + .expect("should parse response") + } + + #[test] + fn posts_consent_and_network_signals_to_the_partner_url() { + let provider = provider(); + let consent = ConsentContext { + raw_us_privacy: Some("1YNN".to_owned()), + gpp_section_ids: Some(vec![7, 8]), + ..ConsentContext::default() + }; + let lookup = LookupInput { + ec_id: "ec-example", + consent: &consent, + client: ClientSignals { + ip: Some("192.0.2.10".parse().expect("should parse ip")), + user_agent: Some("ExampleBrowser/1.0"), + }, + publisher_domain: "www.example.com", + due_sources: &[], + current: OwnedEntries::new(None, &[]), + hashed_email: None, + }; + + let request = provider + .build_request(&lookup) + .expect("should build request"); + let body: serde_json::Value = + serde_json::from_slice(&request.body).expect("should send JSON"); + + assert_eq!(request.method, Method::POST, "should post"); + assert_eq!( + request.url.as_str(), + "https://id5.example.com/g/v2/1234.json", + "should call the partner's fetch URL" + ); + assert_eq!( + request.headers.get(ORIGIN).map(HeaderValue::as_bytes), + Some(&b"https://www.example.com"[..]), + "should send the publisher origin" + ); + assert_eq!(body["partner"], 1234, "should send the partner"); + assert_eq!(body["gdpr"], 0, "should send that GDPR does not apply"); + assert_eq!(body["us_privacy"], "1YNN", "should send US privacy"); + assert_eq!(body["gpp_sid"], "7,8", "should join GPP section IDs"); + assert_eq!(body["ip"], "192.0.2.10", "should send the client IP"); + assert_eq!( + body["ua"], "ExampleBrowser/1.0", + "should send the user agent" + ); + } + + #[test] + fn drops_a_private_client_ip() { + let provider = provider(); + let consent = ConsentContext::default(); + let lookup = LookupInput { + ec_id: "ec-example", + consent: &consent, + client: ClientSignals { + ip: Some("10.1.2.3".parse().expect("should parse ip")), + user_agent: None, + }, + publisher_domain: "www.example.com", + due_sources: &[], + current: OwnedEntries::new(None, &[]), + hashed_email: None, + }; + + let request = provider + .build_request(&lookup) + .expect("should build request"); + let body: serde_json::Value = + serde_json::from_slice(&request.body).expect("should send JSON"); + + assert!(body.get("ip").is_none(), "should not send a private IP"); + } + + #[test] + fn parses_complete_eids_for_configured_sources() { + let provider = provider(); + let headers = HeaderMap::new(); + let body = serde_json::to_vec(&json!({ + "ids": { + "id5": { "eid": { "source": "id5.example.com", "uids": [{ "id": "id5-example", "atype": 1, "ext": { "linkType": 2 } }] } }, + "other": { "eid": { "source": "unknown.example.com", "uids": [{ "id": "x" }] } }, + }, + "cache_control": { "max_age_sec": 3600 }, + })) + .expect("should encode body"); + + let LookupOutcome::Found(eids) = respond(&*provider, &headers, &body) else { + panic!("should find an EID"); + }; + + assert_eq!(eids.len(), 1, "should drop unconfigured sources"); + assert_eq!(eids[0].source, "id5.example.com", "should keep the source"); + assert_eq!(eids[0].uids[0].id, "id5-example", "should keep the UID"); + assert!(eids[0].uids[0].ext.is_some(), "should keep the UID ext"); + assert!( + matches!(eids[0].uids[0].expiry, ProviderExpiry::After(d) if d == Duration::from_secs(3600)), + "should use the cache max age" + ); + } + + #[test] + fn falls_back_to_the_legacy_universal_uid() { + let provider = provider(); + let headers = HeaderMap::new(); + let body = br#"{"universal_uid":"legacy-example"}"#; + + let LookupOutcome::Found(eids) = respond(&*provider, &headers, body) else { + panic!("should find an EID"); + }; + + assert_eq!( + eids[0].source, "id5.example.com", + "should use the first source" + ); + assert_eq!(eids[0].uids[0].id, "legacy-example", "should keep the UID"); + } + + #[test] + fn reports_no_match_for_an_opted_out_or_empty_reply() { + let provider = provider(); + let headers = HeaderMap::new(); + + for body in [&br#"{"universal_uid":"0"}"#[..], b"{}"] { + assert!( + matches!(respond(&*provider, &headers, body), LookupOutcome::NoMatch), + "should report no match" + ); + } + } + + #[test] + fn rejects_an_error_status_and_a_malformed_body() { + let provider = provider(); + let headers = HeaderMap::new(); + + for (status, body) in [ + (StatusCode::INTERNAL_SERVER_ERROR, &b"{}"[..]), + (StatusCode::OK, b"not json"), + ] { + let error = provider + .parse_response(ProviderResponse { + status, + headers: &headers, + body, + }) + .expect_err("should reject the response"); + assert!( + matches!(error.current_context(), ProviderError::InvalidResponse), + "should report an invalid response" + ); + } + } +} diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index e11e69abf..0b30a5a23 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -9,12 +9,17 @@ //! Dependencies point one way: this crate depends on core, and core never //! depends on it. +mod common; +pub mod id5; + use std::sync::Arc; use std::time::Duration; use error_stack::Report; use trusted_server_core::ec::coordinator::{ProviderCoordinator, RegisteredProvider}; -use trusted_server_core::ec::provider::{IdentityProvider, ProviderConfigError}; +use trusted_server_core::ec::provider::{ + IdentityProvider, ProviderConfigError, ProviderKind as _, build_kind, +}; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::TrustedServerError; use trusted_server_core::settings::{ @@ -23,10 +28,9 @@ use trusted_server_core::settings::{ /// Whether `kind` has a registered implementation. /// -/// No kinds are registered yet. #[must_use] -pub fn is_registered_kind(_kind: &str) -> bool { - false +pub fn is_registered_kind(kind: &str) -> bool { + matches!(kind, id5::Id5::KIND) } /// Builds the partner registry, marking the sources claimed by enabled @@ -42,19 +46,19 @@ pub fn partner_registry(ec: &Ec) -> Result Result, Report> { - let kind = config.kind(); - Err(Report::new(ProviderConfigError::UnknownKind) - .attach(format!("kind `{kind}` is not registered"))) + match config.kind() { + id5::Id5::KIND => build_kind::(config, secrets), + kind => Err(Report::new(ProviderConfigError::UnknownKind) + .attach(format!("kind `{kind}` is not registered"))), + } } /// Builds the coordinator for enabled providers collected before the diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-id5.md b/docs/superpowers/plans/2026-11-02-identity-provider-id5.md new file mode 100644 index 000000000..68fcc116e --- /dev/null +++ b/docs/superpowers/plans/2026-11-02-identity-provider-id5.md @@ -0,0 +1,38 @@ +# ID5 Identity Provider Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Resolve the ID5 ID at the edge as an anonymous `[[ec.identity_providers]]` entry with `kind = "id5"`, storing the complete EIDs ID5 returns. + +**Architecture:** `crates/trusted-server-identity-providers/src/id5.rs` holds `Id5` (a `ProviderKind`), `Id5Settings` and `Id5Provider` (an `IdentityProvider`), plus one `build` arm in `lib.rs`. The provider is pure: `build_request` builds one JSON `POST`, and `parse_response` borrows EIDs from the body. The coordinator from PR A (`ec::coordinator`) applies consent, the source gate, validation, expiry and persistence. + +**Spec:** `docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md`, section 7. + +**Changes from the earlier ID5 plan (built on the old descriptor framework):** + +- No hashed email and no `pd` field: every source is `SourceInput::Anonymous`. +- No private provider state, so ID5's `signature` is not sent back. +- No page URL in `LookupInput`, so `tml` is the publisher origin. +- Endpoint, sources and the TCF vendor ID come from the common provider config, not hard-coded defaults. +- The legacy `universal_uid` fallback goes to the first configured source. An opted-out `universal_uid` of `0` is reported as `LookupOutcome::NoMatch`; there is no separate invalidation outcome. + +## Request + +- `POST /.json`, with `Content-Type: application/json` and `Origin: https://`. +- Body: `partner`, `v`, `o = "trusted-server"`, `nbPage = 1`, `gdpr`, and when present `gdpr_consent`, `us_privacy`, `gpp_string`, `gpp_sid` (comma-joined), `ua`, and `ip` (only when routable, see `common::is_routable_ip`). +- Confirm the `ip` and `ua` field names with ID5 for each partner account. + +## Response + +- A non-2xx status or an unparseable body is `ProviderError::InvalidResponse`. +- `ids..eid` entries whose source is configured become `ProviderEid`s; every field, including `ext`, is borrowed from the body. +- `cache_control.max_age_sec` becomes `ProviderExpiry::After` for every UID. +- With no EID for the first source, a non-empty, non-`0` `universal_uid` becomes an EID with `atype = 1`. + +## Tasks + +- [ ] Add `common.rs` with `is_routable_ip`, `join_section_ids` and the borrowed `VendorEid`. +- [ ] Add `id5.rs` and the `build` arm. +- [ ] Unit tests: request shape, private IP dropped, complete EIDs parsed and unconfigured sources dropped, legacy fallback, opted-out and empty replies are no match, error status and malformed body rejected. +- [ ] Add a commented `kind = "id5"` example to `trusted-server.example.toml`. +- [ ] Run `cargo test-fastly` and `cargo clippy-fastly`. diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 7bf4545a1..6e3eb3fa1 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -150,6 +150,19 @@ pull_sync_concurrency = 3 # token = "example_provider_token" # Provider-specific settings, parsed by the provider kind. # [ec.identity_providers.settings] +# +# ID5 (kind "id5"): anonymous lookup, no secrets. The first source receives +# the legacy `universal_uid`. Use the endpoint and sources ID5 gives you. +# [[ec.identity_providers]] +# id = "id5" +# kind = "id5" +# enabled = false +# endpoint = "https://id5.example.com/g/v2" +# allowed_domains = ["id5.example.com"] +# sources = ["id5.example.com"] +# tcf_vendor_id = 9999 +# [ec.identity_providers.settings] +# partner_id = 1234 # ============================================================================= From bdbbeabe459f22da2c6e8a28362a9a91d3971c88 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 04:37:10 +0530 Subject: [PATCH 02/72] Add the LiveIntent identity provider Signed-off-by: dhruv8sh --- Cargo.lock | 2 + .../Cargo.toml | 2 + .../src/lib.rs | 4 +- .../src/liveintent.rs | 433 ++++++++++++++++++ ...2026-11-02-identity-provider-liveintent.md | 42 ++ trusted-server.example.toml | 17 + 6 files changed, 499 insertions(+), 1 deletion(-) create mode 100644 crates/trusted-server-identity-providers/src/liveintent.rs create mode 100644 docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md diff --git a/Cargo.lock b/Cargo.lock index e243b3c3a..4d558019e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6742,9 +6742,11 @@ dependencies = [ name = "trusted-server-identity-providers" version = "0.1.0" dependencies = [ + "base64", "bytes", "error-stack", "http", + "httpdate", "log", "serde", "serde_json", diff --git a/crates/trusted-server-identity-providers/Cargo.toml b/crates/trusted-server-identity-providers/Cargo.toml index aac7daab5..e01ab9378 100644 --- a/crates/trusted-server-identity-providers/Cargo.toml +++ b/crates/trusted-server-identity-providers/Cargo.toml @@ -14,9 +14,11 @@ doctest = false workspace = true [dependencies] +base64 = { workspace = true } bytes = { workspace = true } error-stack = { workspace = true } http = { workspace = true } +httpdate = { workspace = true } log = { workspace = true } serde = { workspace = true } serde_json = { workspace = true, features = ["raw_value"] } diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index 0b30a5a23..f1ea82c3b 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -11,6 +11,7 @@ mod common; pub mod id5; +pub mod liveintent; use std::sync::Arc; use std::time::Duration; @@ -30,7 +31,7 @@ use trusted_server_core::settings::{ /// #[must_use] pub fn is_registered_kind(kind: &str) -> bool { - matches!(kind, id5::Id5::KIND) + matches!(kind, id5::Id5::KIND | liveintent::LiveIntent::KIND) } /// Builds the partner registry, marking the sources claimed by enabled @@ -56,6 +57,7 @@ pub fn build( ) -> Result, Report> { match config.kind() { id5::Id5::KIND => build_kind::(config, secrets), + liveintent::LiveIntent::KIND => build_kind::(config, secrets), kind => Err(Report::new(ProviderConfigError::UnknownKind) .attach(format!("kind `{kind}` is not registered"))), } diff --git a/crates/trusted-server-identity-providers/src/liveintent.rs b/crates/trusted-server-identity-providers/src/liveintent.rs new file mode 100644 index 000000000..c8f547bbf --- /dev/null +++ b/crates/trusted-server-identity-providers/src/liveintent.rs @@ -0,0 +1,433 @@ +//! `LiveIntent` identity provider: resolves `LiveIntent`'s IDs server side. +//! +//! Calls `LiveIntent`'s identity exchange with consent and network signals +//! and asks for the attributes the operator configured. Each attribute maps +//! to one configured `OpenRTB` source. The lookup is anonymous: no hashed +//! email and no first-party `LiveIntent` cookie are sent. + +use std::borrow::Cow; +use std::collections::{BTreeMap, HashMap}; +use std::num::NonZeroUsize; +use std::time::UNIX_EPOCH; + +use base64::Engine as _; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use bytes::Bytes; +use error_stack::{Report, ResultExt as _}; +use http::header::{EXPIRES, ORIGIN}; +use http::{HeaderMap, HeaderValue, Method, StatusCode}; +use serde::Deserialize; +use serde_json::value::RawValue; +use trusted_server_core::ec::eid_types::{SourceDomain, UnixSeconds}; +use trusted_server_core::ec::provider::{ + EidLevelFields, EidShape, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, + ProviderCommon, ProviderConfigError, ProviderEid, ProviderError, ProviderExpiry, ProviderKind, + ProviderRequest, ProviderResponse, ProviderUid, SourceCapability, SourceInput, +}; +use url::Url; + +use crate::common::{is_routable_ip, join_section_ids}; + +/// `atype` `LiveIntent` IDs carry: a person-based ID. +const LIVEINTENT_ATYPE: i32 = 3; + +/// Most UIDs accepted per `LiveIntent` EID. +const MAX_UIDS_PER_EID: NonZeroUsize = NonZeroUsize::new(1).expect("should be non-zero"); + +/// The `LiveIntent` provider kind. +pub struct LiveIntent; + +/// Settings for a `liveintent` provider. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LiveIntentSettings { + /// The `LiveIntent` publisher ID; `LiveIntent` accepts `any`. + #[serde(default = "default_publisher_id")] + pub publisher_id: String, + /// The distributor ID, when the publisher has one. + #[serde(default)] + pub distributor_id: Option, + /// Map of `LiveIntent` attribute name to the source it is stored under. + /// + /// Every source must be one of the provider's configured `sources`. + pub attributes: BTreeMap, +} + +fn default_publisher_id() -> String { + "any".to_owned() +} + +/// A built `LiveIntent` provider. +#[derive(Debug)] +pub struct LiveIntentProvider { + capabilities: ProviderCapabilities, + url: Url, + distributor_id: Option, + attributes: Box<[(String, SourceDomain)]>, +} + +impl ProviderKind for LiveIntent { + const KIND: &'static str = "liveintent"; + type Settings = LiveIntentSettings; + type Provider = LiveIntentProvider; + + fn build( + common: &ProviderCommon<'_>, + settings: LiveIntentSettings, + ) -> Result> { + let attributes = settings + .attributes + .into_iter() + .map(|(attribute, source)| { + let source = common + .sources + .iter() + .find(|known| known.as_str() == source) + .ok_or_else(|| { + Report::new(ProviderConfigError::Invalid).attach(format!( + "attribute `{attribute}` maps to `{source}`, which is not a configured source" + )) + })?; + Ok((attribute, source.clone())) + }) + .collect::, Report>>()?; + let max_eids = NonZeroUsize::new(attributes.len()).ok_or_else(|| { + Report::new(ProviderConfigError::Invalid) + .attach("a liveintent provider needs at least one attribute") + })?; + let distributor = settings.distributor_id.as_deref().unwrap_or("prebid"); + let mut url = common.endpoint.clone(); + url.path_segments_mut() + .map_err(|()| { + Report::new(ProviderConfigError::Invalid).attach("endpoint cannot hold a path") + })? + .pop_if_empty() + .push(distributor) + .push(&settings.publisher_id); + Ok(LiveIntentProvider { + capabilities: ProviderCapabilities { + sources: attributes + .iter() + .map(|(_, source)| SourceCapability { + source: source.clone(), + input: SourceInput::Anonymous, + }) + .collect(), + shape: EidShape { + max_eids, + max_uids_per_eid: MAX_UIDS_PER_EID, + eid_level_fields: EidLevelFields::Ignored, + }, + }, + url, + distributor_id: settings.distributor_id, + attributes, + }) + } +} + +/// Reads the `Expires` header as an absolute expiry. +fn expiry(headers: &HeaderMap) -> ProviderExpiry { + headers + .get(EXPIRES) + .and_then(|value| value.to_str().ok()) + .and_then(|value| httpdate::parse_http_date(value).ok()) + .and_then(|at| at.duration_since(UNIX_EPOCH).ok()) + .map_or(ProviderExpiry::Unspecified, |since| { + ProviderExpiry::At(UnixSeconds::new(since.as_secs())) + }) +} + +impl IdentityProvider for LiveIntentProvider { + fn capabilities(&self) -> &ProviderCapabilities { + &self.capabilities + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + let consent = lookup.consent; + let origin = format!("https://{}", lookup.publisher_domain); + let mut url = self.url.clone(); + { + let mut query = url.query_pairs_mut(); + if let Some(distributor) = &self.distributor_id { + query.append_pair("did", distributor); + } + query.append_pair("gdpr", if consent.gdpr_applies { "1" } else { "0" }); + if let Some(tc_string) = &consent.raw_tc_string { + query.append_pair("gdpr_consent", tc_string); + } + if let Some(us_privacy) = &consent.raw_us_privacy { + query.append_pair("us_privacy", us_privacy); + } + if let Some(gpp) = &consent.raw_gpp_string { + query.append_pair("gpp_s", gpp); + } + if let Some(ids) = &consent.gpp_section_ids { + query.append_pair("gpp_as", &join_section_ids(ids)); + } + query.append_pair("pu", &origin); + if let Some(ip) = lookup.client.ip.filter(|ip| is_routable_ip(*ip)) { + let name = if ip.is_ipv4() { "pip" } else { "pip6" }; + query.append_pair(name, &URL_SAFE_NO_PAD.encode(ip.to_string())); + } + for (attribute, source) in &self.attributes { + if lookup.due_sources.contains(source) { + query.append_pair("resolve", attribute); + } + } + } + let mut headers = HeaderMap::new(); + headers.insert( + ORIGIN, + HeaderValue::from_str(&origin).change_context(ProviderError::Request)?, + ); + Ok(ProviderRequest { + method: Method::GET, + url, + headers, + body: Bytes::new(), + }) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + if response.status == StatusCode::NO_CONTENT { + return Ok(LookupOutcome::NoMatch); + } + if !response.status.is_success() { + return Err(Report::new(ProviderError::InvalidResponse) + .attach(format!("LiveIntent returned status {}", response.status))); + } + let reply: HashMap<&'body str, &'body RawValue> = + serde_json::from_slice(response.body).change_context(ProviderError::InvalidResponse)?; + let expiry = expiry(response.headers); + let eids: Vec> = self + .attributes + .iter() + .filter_map(|(attribute, source)| { + let raw = reply.get(attribute.as_str())?; + let id = Cow::<'body, str>::deserialize(*raw) + .ok() + .filter(|id| !id.is_empty())?; + Some(ProviderEid { + source: Cow::Owned(source.as_str().to_owned()), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids: vec![ProviderUid { + id, + atype: Some(LIVEINTENT_ATYPE), + ext: None, + expiry, + }], + }) + }) + .collect(); + Ok(if eids.is_empty() { + LookupOutcome::NoMatch + } else { + LookupOutcome::Found(eids) + }) + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + use trusted_server_core::consent::types::ConsentContext; + use trusted_server_core::ec::provider::{ClientSignals, OwnedEntries, build_kind}; + use trusted_server_core::settings::{IdentityProviderConfig, ProviderSecrets}; + + use super::*; + + fn config(attributes: &serde_json::Value) -> IdentityProviderConfig { + serde_json::from_value(json!({ + "id": "liveintent", + "kind": "liveintent", + "endpoint": "https://li.example.com/idex", + "allowed_domains": ["li.example.com"], + "sources": ["li.example.com", "li-partner.example.com"], + "settings": { "publisher_id": "12345", "attributes": attributes }, + })) + .expect("should parse provider config") + } + + fn provider() -> Box { + build_kind::( + &config(&json!({ "nonId": "li.example.com", "partner": "li-partner.example.com" })), + &ProviderSecrets::default(), + ) + .expect("should build provider") + } + + fn due() -> Vec { + ["li.example.com", "li-partner.example.com"] + .into_iter() + .map(|source| SourceDomain::try_from(source).expect("should parse source")) + .collect() + } + + #[test] + fn rejects_an_attribute_mapped_to_an_unconfigured_source() { + let error = build_kind::( + &config(&json!({ "nonId": "other.example.com" })), + &ProviderSecrets::default(), + ) + .err() + .expect("should reject the config"); + + assert!( + matches!(error.current_context(), ProviderConfigError::Invalid), + "should report invalid config" + ); + } + + #[test] + fn calls_the_exchange_with_consent_ip_and_due_attributes() { + let provider = provider(); + let consent = ConsentContext { + raw_us_privacy: Some("1YNN".to_owned()), + ..ConsentContext::default() + }; + let due = due(); + let lookup = LookupInput { + ec_id: "ec-example", + consent: &consent, + client: ClientSignals { + ip: Some("192.0.2.10".parse().expect("should parse ip")), + user_agent: None, + }, + publisher_domain: "www.example.com", + due_sources: &due[..1], + current: OwnedEntries::new(None, &[]), + hashed_email: None, + }; + + let request = provider + .build_request(&lookup) + .expect("should build request"); + let query: Vec<(String, String)> = request + .url + .query_pairs() + .map(|(key, value)| (key.into_owned(), value.into_owned())) + .collect(); + let value = |name: &str| { + query + .iter() + .filter(|(key, _)| key == name) + .map(|(_, value)| value.as_str()) + .collect::>() + }; + + assert_eq!(request.method, Method::GET, "should use GET"); + assert_eq!( + request.url.path(), + "/idex/prebid/12345", + "should call the distributor and publisher path" + ); + assert_eq!(value("gdpr"), ["0"], "should send that GDPR does not apply"); + assert_eq!(value("us_privacy"), ["1YNN"], "should send US privacy"); + assert_eq!( + value("pu"), + ["https://www.example.com"], + "should send the origin" + ); + assert_eq!( + value("pip"), + [URL_SAFE_NO_PAD.encode("192.0.2.10").as_str()], + "should send the encoded IP" + ); + assert_eq!( + value("resolve"), + ["nonId"], + "should ask only for due attributes" + ); + } + + #[test] + fn maps_returned_attributes_to_sources_with_the_expires_header() { + let provider = provider(); + let mut headers = HeaderMap::new(); + headers.insert( + EXPIRES, + HeaderValue::from_static("Sun, 06 Nov 1994 08:49:37 GMT"), + ); + let body = br#"{"nonId":"li-example","partner":"partner-example","unknown":"x"}"#; + + let outcome = provider + .parse_response(ProviderResponse { + status: StatusCode::OK, + headers: &headers, + body, + }) + .expect("should parse response"); + let LookupOutcome::Found(eids) = outcome else { + panic!("should find EIDs"); + }; + + assert_eq!(eids.len(), 2, "should return each configured attribute"); + assert_eq!(eids[0].source, "li.example.com", "should map nonId"); + assert_eq!(eids[0].uids[0].id, "li-example", "should keep the ID"); + assert_eq!(eids[0].uids[0].atype, Some(3), "should set atype 3"); + assert!( + matches!(eids[0].uids[0].expiry, ProviderExpiry::At(at) if at.get() == 784_111_777), + "should use the Expires header" + ); + assert_eq!( + eids[1].source, "li-partner.example.com", + "should map partner" + ); + } + + #[test] + fn reports_no_match_for_no_content_and_empty_ids() { + let provider = provider(); + let headers = HeaderMap::new(); + + for (status, body) in [ + (StatusCode::NO_CONTENT, &b""[..]), + (StatusCode::OK, br#"{"nonId":""}"#), + ] { + let outcome = provider + .parse_response(ProviderResponse { + status, + headers: &headers, + body, + }) + .expect("should parse response"); + assert!( + matches!(outcome, LookupOutcome::NoMatch), + "should report no match" + ); + } + } + + #[test] + fn rejects_an_error_status_and_a_malformed_body() { + let provider = provider(); + let headers = HeaderMap::new(); + + for (status, body) in [ + (StatusCode::BAD_GATEWAY, &b"{}"[..]), + (StatusCode::OK, b"not json"), + ] { + let error = provider + .parse_response(ProviderResponse { + status, + headers: &headers, + body, + }) + .expect_err("should reject the response"); + assert!( + matches!(error.current_context(), ProviderError::InvalidResponse), + "should report an invalid response" + ); + } + } +} diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md b/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md new file mode 100644 index 000000000..94d1026cf --- /dev/null +++ b/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md @@ -0,0 +1,42 @@ +# LiveIntent Identity Provider Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Resolve `LiveIntent`'s IDs at the edge as an anonymous `[[ec.identity_providers]]` entry with `kind = "liveintent"`, storing one EID per configured attribute. + +**Architecture:** `crates/trusted-server-identity-providers/src/liveintent.rs` holds `LiveIntent` (a `ProviderKind`), `LiveIntentSettings` and `LiveIntentProvider` (an `IdentityProvider`), plus one `build` arm in `lib.rs`. The provider is pure: `build_request` builds one `GET` to the identity exchange, and `parse_response` maps returned attributes to sources. The coordinator from PR A applies consent, the source gate, validation, expiry and persistence. + +**Spec:** `docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md`, section 7. + +**Changes from the earlier LiveIntent plan (built on the old descriptor framework):** + +- Anonymous only: no hashed email, and no first-party `LiveIntent` cookie (`duid`), because `LookupInput` carries no cookies. +- The attribute-to-source table is operator configuration (`settings.attributes`), not a hard-coded list; every mapped source must be one of the provider's `sources`. +- No page URL in `LookupInput`, so `pu` is the publisher origin. +- EID-level fields are ignored and no `ext` is synthesized (`segments`, `provider`, `rtiPartner`), because `ProviderUid::ext` borrows from the response body. +- The `Expires` header becomes `ProviderExpiry::At`; the provider never reads the clock. + +## Settings + +| Key | Default | Meaning | +| ---------------- | ------------- | ------------------------------------------------- | +| `publisher_id` | `any` | Path segment after the distributor. | +| `distributor_id` | none | Sent as `did`; the path uses `prebid` when unset. | +| `attributes` | required, ≥ 1 | `LiveIntent` attribute name → stored source. | + +## Request + +- `GET //` with `Origin: https://`. +- Query: `did`, `gdpr`, `gdpr_consent`, `us_privacy`, `gpp_s`, `gpp_as`, `pu`, `pip` or `pip6` (base64url of a routable client IP), and one `resolve` per attribute whose source is due. + +## Response + +- `204 No Content` is `LookupOutcome::NoMatch`; another non-2xx status or a non-object body is `ProviderError::InvalidResponse`. +- Each configured attribute with a non-empty string value becomes one EID with `atype = 3`, borrowed from the body. + +## Tasks + +- [ ] Add `liveintent.rs` and the `build` arm. +- [ ] Unit tests: unconfigured mapped source rejected, request shape with only due attributes, attributes mapped with the `Expires` expiry, no-content and empty IDs are no match, error status and malformed body rejected. +- [ ] Add a commented `kind = "liveintent"` example to `trusted-server.example.toml`. +- [ ] Run `cargo test-fastly` and `cargo clippy-fastly`. diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 6e3eb3fa1..66f2826f2 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -163,6 +163,23 @@ pull_sync_concurrency = 3 # tcf_vendor_id = 9999 # [ec.identity_providers.settings] # partner_id = 1234 +# +# LiveIntent (kind "liveintent"): anonymous lookup, no secrets. Each attribute +# LiveIntent returns is stored under the source it maps to; every mapped +# source must be listed in `sources`. +# [[ec.identity_providers]] +# id = "liveintent" +# kind = "liveintent" +# enabled = false +# endpoint = "https://liveintent.example.com/idex" +# allowed_domains = ["liveintent.example.com"] +# sources = ["liveintent.example.com"] +# tcf_vendor_id = 9999 +# [ec.identity_providers.settings] +# publisher_id = "any" +# distributor_id = "example-distributor" +# [ec.identity_providers.settings.attributes] +# nonId = "liveintent.example.com" # ============================================================================= From d62b5b33d9d8906997cc130b74275ca787e75632 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 12:59:08 +0530 Subject: [PATCH 03/72] Fix vendor expiry, opt-out and parsing issues Signed-off-by: dhruv8sh --- .../src/common.rs | 164 +++++++++++++++--- .../src/id5.rs | 133 ++++++++++---- .../src/liveintent.rs | 85 ++++++--- 3 files changed, 308 insertions(+), 74 deletions(-) diff --git a/crates/trusted-server-identity-providers/src/common.rs b/crates/trusted-server-identity-providers/src/common.rs index ff7581d8d..5d37e39fb 100644 --- a/crates/trusted-server-identity-providers/src/common.rs +++ b/crates/trusted-server-identity-providers/src/common.rs @@ -1,7 +1,8 @@ //! Helpers shared by the vendor provider modules. use std::borrow::Cow; -use std::net::IpAddr; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::time::Duration; use serde::Deserialize; use serde_json::value::RawValue; @@ -9,31 +10,97 @@ use trusted_server_core::ec::provider::{ProviderEid, ProviderExpiry, ProviderUid /// Returns whether `ip` is a public address worth sending to a vendor. /// -/// Loopback, private, link-local, carrier-grade NAT and unique-local -/// addresses identify nothing outside the operator's network. +/// Drops every special-purpose range that identifies nothing outside the +/// operator's network or is never a real client: for IPv4 "this network" +/// `0.0.0.0/8`, private, carrier-grade NAT `100.64.0.0/10`, loopback, +/// link-local, IETF protocol assignments `192.0.0.0/24`, the TEST-NETs, +/// the 6to4 relay anycast `192.88.99.0/24`, benchmarking `198.18.0.0/15`, +/// multicast and the reserved `240.0.0.0/4` (including broadcast). For IPv6 +/// it drops unspecified, loopback, IPv4-compatible, discard-only +/// `100::/64`, IETF protocol assignments `2001::/23`, documentation +/// `2001:db8::/32`, unique-local `fc00::/7`, link-local `fe80::/10`, +/// site-local `fec0::/10` and multicast. IPv4-mapped and NAT64 +/// `64:ff9b::/96` addresses are judged by their embedded IPv4 address. #[must_use] pub(crate) fn is_routable_ip(ip: IpAddr) -> bool { match ip { - IpAddr::V4(v4) => { - let [first, second, ..] = v4.octets(); - let carrier_grade_nat = first == 100 && (second & 0xc0) == 64; - !(v4.is_loopback() - || v4.is_unspecified() - || v4.is_private() - || v4.is_link_local() - || v4.is_broadcast() - || carrier_grade_nat) - } - IpAddr::V6(v6) => { - if let Some(v4) = v6.to_ipv4_mapped() { - return is_routable_ip(IpAddr::V4(v4)); - } - let first = v6.segments()[0]; - let unique_local = (first & 0xfe00) == 0xfc00; - let link_local = (first & 0xffc0) == 0xfe80; - !(v6.is_loopback() || v6.is_unspecified() || unique_local || link_local) - } + IpAddr::V4(v4) => is_routable_ipv4(v4), + IpAddr::V6(v6) => is_routable_ipv6(v6), + } +} + +fn is_routable_ipv4(v4: Ipv4Addr) -> bool { + let [first, second, third, _] = v4.octets(); + let this_network = first == 0; + let carrier_grade_nat = first == 100 && (second & 0xc0) == 64; + let protocol_assignments = first == 192 && second == 0 && third == 0; + let test_net = matches!( + (first, second, third), + (192, 0, 2) | (198, 51, 100) | (203, 0, 113) + ); + let relay_anycast = first == 192 && second == 88 && third == 99; + let benchmarking = first == 198 && (second & 0xfe) == 18; + let multicast = (first & 0xf0) == 224; + let reserved = (first & 0xf0) == 240; + !(this_network + || v4.is_loopback() + || v4.is_private() + || v4.is_link_local() + || carrier_grade_nat + || protocol_assignments + || test_net + || relay_anycast + || benchmarking + || multicast + || reserved) +} + +fn is_routable_ipv6(v6: Ipv6Addr) -> bool { + if let Some(v4) = v6.to_ipv4_mapped() { + return is_routable_ipv4(v4); } + let segments = v6.segments(); + if segments[..6] == [0x64, 0xff9b, 0, 0, 0, 0] { + let [.., a, b, c, d] = v6.octets(); + return is_routable_ipv4(Ipv4Addr::new(a, b, c, d)); + } + let first = segments[0]; + let ipv4_compatible = segments[..6] == [0; 6]; + let discard_only = segments[..4] == [0x100, 0, 0, 0]; + let protocol_assignments = first == 0x2001 && segments[1] < 0x200; + let documentation = first == 0x2001 && segments[1] == 0x0db8; + let unique_local = (first & 0xfe00) == 0xfc00; + let link_local = (first & 0xffc0) == 0xfe80; + let site_local = (first & 0xffc0) == 0xfec0; + let multicast = (first & 0xff00) == 0xff00; + !(ipv4_compatible + || discard_only + || protocol_assignments + || documentation + || unique_local + || link_local + || site_local + || multicast) +} + +/// TTL given to a vendor UID whose response states no expiry. +pub(crate) const DEFAULT_TTL: Duration = Duration::from_secs(24 * 60 * 60); + +/// Shortest TTL a vendor UID is kept for. +pub(crate) const MIN_TTL: Duration = Duration::from_secs(60 * 60); + +/// Longest TTL a vendor UID is kept for. +pub(crate) const MAX_TTL: Duration = Duration::from_secs(30 * 24 * 60 * 60); + +/// Turns a vendor-stated TTL into a bounded relative expiry. +/// +/// A missing TTL becomes [`DEFAULT_TTL`] so a UID never lives forever. A +/// stated TTL is clamped to [`MIN_TTL`]..=[`MAX_TTL`]: a zero, past or very +/// short TTL keeps the UID for [`MIN_TTL`] instead of discarding a valid +/// ID, and the minimum still forces a refresh soon after. +#[must_use] +pub(crate) fn bounded_expiry(ttl: Option) -> ProviderExpiry { + ProviderExpiry::After(ttl.map_or(DEFAULT_TTL, |ttl| ttl.clamp(MIN_TTL, MAX_TTL))) } /// Joins GPP section IDs with commas, as vendors expect them. @@ -101,23 +168,57 @@ mod tests { use super::*; #[test] - fn keeps_public_addresses_and_drops_local_ones() { - for ip in ["192.0.2.10", "198.51.100.7", "2001:db8::1"] { + fn keeps_public_addresses() { + for ip in [ + "11.22.33.44", + "203.0.112.9", + "2a00:1450::1", + "::ffff:11.22.33.44", + ] { assert!( is_routable_ip(ip.parse().expect("should parse")), "should keep {ip}" ); } + } + + #[test] + fn drops_special_purpose_addresses() { for ip in [ + "0.1.2.3", "127.0.0.1", "10.1.2.3", + "172.16.0.1", "192.168.1.1", "169.254.1.1", "100.64.0.1", + "100.127.255.254", + "192.0.0.8", + "192.0.2.10", + "198.51.100.7", + "203.0.113.5", + "192.88.99.1", + "198.18.0.1", + "198.19.255.1", + "224.0.0.1", + "239.1.1.1", + "240.0.0.1", + "255.255.255.255", + "::", "::1", + "::10.1.2.3", + "100::1", + "2001::1", + "2001:db8::1", + "fc00::1", "fd00::1", "fe80::1", + "febf::1", + "fec0::1", + "ff02::1", "::ffff:127.0.0.1", + "::ffff:100.64.0.1", + "64:ff9b::10.1.2.3", ] { assert!( !is_routable_ip(ip.parse().expect("should parse")), @@ -125,4 +226,19 @@ mod tests { ); } } + + #[test] + fn bounds_vendor_expiry() { + for (ttl, expected) in [ + (None, DEFAULT_TTL), + (Some(Duration::ZERO), MIN_TTL), + (Some(Duration::from_secs(7200)), Duration::from_secs(7200)), + (Some(Duration::from_secs(u64::MAX)), MAX_TTL), + ] { + assert!( + matches!(bounded_expiry(ttl), ProviderExpiry::After(after) if after == expected), + "should bound {ttl:?} to {expected:?}" + ); + } + } } diff --git a/crates/trusted-server-identity-providers/src/id5.rs b/crates/trusted-server-identity-providers/src/id5.rs index 3115e9141..036d6eb69 100644 --- a/crates/trusted-server-identity-providers/src/id5.rs +++ b/crates/trusted-server-identity-providers/src/id5.rs @@ -5,6 +5,12 @@ //! under `ids`, or the legacy `universal_uid`, which is given to the first //! configured source. //! +//! Parsing is lenient per entry: an entry of `ids`, `universal_uid` or +//! `cache_control` with an unexpected shape is skipped rather than failing +//! the whole response. The opt-out UID `"0"` is never stored or forwarded. +//! `cache_control.max_age_sec` is bounded by [`bounded_expiry`]: a missing +//! value gets the default TTL and a zero or tiny one the minimum TTL. +//! //! The request field names for the client IP and user agent follow ID5's //! server-to-server integration and should be confirmed with ID5 for each //! partner account. @@ -22,14 +28,16 @@ use serde::{Deserialize, Serialize}; use trusted_server_core::ec::eid_types::SourceDomain; use trusted_server_core::ec::provider::{ EidLevelFields, EidShape, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, - ProviderCommon, ProviderConfigError, ProviderEid, ProviderError, ProviderExpiry, ProviderKind, - ProviderRequest, ProviderResponse, ProviderUid, SourceCapability, SourceInput, + ProviderCommon, ProviderConfigError, ProviderEid, ProviderError, ProviderKind, ProviderRequest, + ProviderResponse, ProviderUid, SourceCapability, SourceInput, }; use url::Url; -use crate::common::{VendorEid, is_routable_ip, join_section_ids}; +use serde_json::value::RawValue; + +use crate::common::{VendorEid, bounded_expiry, is_routable_ip, join_section_ids}; -/// The `universal_uid` ID5 returns for a visitor who opted out. +/// The UID ID5 returns for a visitor who opted out. const OPTED_OUT_UID: &str = "0"; /// Most EIDs accepted from one ID5 response. @@ -122,14 +130,16 @@ struct FetchBody<'a> { ip: Option, } +/// The top-level reply; each field is parsed separately so one bad shape +/// does not fail the rest. #[derive(Deserialize)] struct FetchReply<'body> { #[serde(default, borrow)] - universal_uid: Option>, + universal_uid: Option<&'body RawValue>, + #[serde(default, borrow)] + ids: Option<&'body RawValue>, #[serde(default, borrow)] - ids: BTreeMap, Id5Id<'body>>, - #[serde(default)] - cache_control: Option, + cache_control: Option<&'body RawValue>, } #[derive(Deserialize)] @@ -148,11 +158,15 @@ impl Id5Provider { &self.capabilities.sources[0].source } - fn is_known(&self, source: &str) -> bool { + /// Returns the configured source matching `source` after normalization + /// (case and trailing dot). + fn known_source(&self, source: &str) -> Option<&SourceDomain> { + let source = SourceDomain::try_from(source).ok()?; self.capabilities .sources .iter() - .any(|known| known.source.as_str() == source) + .map(|known| &known.source) + .find(|known| **known == source) } } @@ -210,26 +224,35 @@ impl IdentityProvider for Id5Provider { } let reply: FetchReply<'body> = serde_json::from_slice(response.body).change_context(ProviderError::InvalidResponse)?; - let expiry = reply + let max_age = reply .cache_control + .and_then(|raw| serde_json::from_str::(raw.get()).ok()) .and_then(|cache| cache.max_age_sec) - .map_or(ProviderExpiry::Unspecified, |seconds| { - ProviderExpiry::After(Duration::from_secs(seconds)) - }); + .map(Duration::from_secs); + let expiry = bounded_expiry(max_age); - let mut eids: Vec> = reply + let entries: BTreeMap = reply .ids + .and_then(|raw| serde_json::from_str(raw.get()).ok()) + .unwrap_or_default(); + let mut eids: Vec> = entries .into_values() + .filter_map(|raw| serde_json::from_str::>(raw.get()).ok()) .filter_map(|id| id.eid) - .filter(|eid| self.is_known(&eid.source)) - .map(|eid| eid.into_provider_eid(expiry)) + .filter_map(|mut eid| { + let source = self.known_source(&eid.source)?; + eid.source = Cow::Owned(source.as_str().to_owned()); + eid.uids.retain(|uid| is_usable_uid(&uid.id)); + (!eid.uids.is_empty()).then(|| eid.into_provider_eid(expiry)) + }) .collect(); let primary = self.primary_source().as_str(); - if !eids.iter().any(|eid| eid.source == primary) - && let Some(uid) = reply - .universal_uid - .filter(|uid| !uid.is_empty() && uid != OPTED_OUT_UID) - { + let has_primary = eids.iter().any(|eid| eid.source == primary); + let universal_uid = reply + .universal_uid + .and_then(|raw| serde_json::from_str::>(raw.get()).ok()) + .filter(|uid| is_usable_uid(uid)); + if !has_primary && let Some(uid) = universal_uid { eids.push(ProviderEid { source: Cow::Owned(primary.to_owned()), inserter: None, @@ -252,12 +275,19 @@ impl IdentityProvider for Id5Provider { } } +/// Returns whether `id` is a real UID: not empty and not the opt-out UID. +fn is_usable_uid(id: &str) -> bool { + !id.is_empty() && id != OPTED_OUT_UID +} + #[cfg(test)] mod tests { use http::StatusCode; use serde_json::json; use trusted_server_core::consent::types::ConsentContext; - use trusted_server_core::ec::provider::{ClientSignals, OwnedEntries, build_kind}; + use trusted_server_core::ec::provider::{ + ClientSignals, OwnedEntries, ProviderExpiry, build_kind, + }; use trusted_server_core::settings::{IdentityProviderConfig, ProviderSecrets}; use super::*; @@ -301,7 +331,7 @@ mod tests { ec_id: "ec-example", consent: &consent, client: ClientSignals { - ip: Some("192.0.2.10".parse().expect("should parse ip")), + ip: Some("11.22.33.44".parse().expect("should parse ip")), user_agent: Some("ExampleBrowser/1.0"), }, publisher_domain: "www.example.com", @@ -331,7 +361,7 @@ mod tests { assert_eq!(body["gdpr"], 0, "should send that GDPR does not apply"); assert_eq!(body["us_privacy"], "1YNN", "should send US privacy"); assert_eq!(body["gpp_sid"], "7,8", "should join GPP section IDs"); - assert_eq!(body["ip"], "192.0.2.10", "should send the client IP"); + assert_eq!(body["ip"], "11.22.33.44", "should send the client IP"); assert_eq!( body["ua"], "ExampleBrowser/1.0", "should send the user agent" @@ -395,9 +425,17 @@ mod tests { fn falls_back_to_the_legacy_universal_uid() { let provider = provider(); let headers = HeaderMap::new(); - let body = br#"{"universal_uid":"legacy-example"}"#; + let body = serde_json::to_vec(&json!({ + "universal_uid": "legacy-example", + "ids": { + "id5": { "eid": { "source": "id5.example.com", "uids": [{ "id": "0" }] } }, + "broken": { "eid": "not an object" }, + }, + "cache_control": "not an object", + })) + .expect("should encode body"); - let LookupOutcome::Found(eids) = respond(&*provider, &headers, body) else { + let LookupOutcome::Found(eids) = respond(&*provider, &headers, &body) else { panic!("should find an EID"); }; @@ -405,7 +443,43 @@ mod tests { eids[0].source, "id5.example.com", "should use the first source" ); + assert_eq!(eids.len(), 1, "should drop the opted-out EID"); assert_eq!(eids[0].uids[0].id, "legacy-example", "should keep the UID"); + assert!( + matches!(eids[0].uids[0].expiry, ProviderExpiry::After(d) if d == crate::common::DEFAULT_TTL), + "should use the default TTL without a cache max age" + ); + } + + #[test] + fn normalizes_sources_skips_bad_entries_and_bounds_expiry() { + let provider = provider(); + let headers = HeaderMap::new(); + let body = serde_json::to_vec(&json!({ + "ids": { + "a": { "eid": { "source": "ID5.Example.com.", "uids": [{ "id": "id5-example" }, { "id": "0" }] } }, + "b": { "eid": { "source": 7 } }, + "c": { "eid": { "source": "id5-link.example.com", "uids": [{ "id": "0" }] } }, + }, + "cache_control": { "max_age_sec": 0 }, + })) + .expect("should encode body"); + + let LookupOutcome::Found(eids) = respond(&*provider, &headers, &body) else { + panic!("should find an EID"); + }; + + assert_eq!(eids.len(), 1, "should skip bad and opted-out entries"); + assert_eq!( + eids[0].source, "id5.example.com", + "should store the configured source" + ); + assert_eq!(eids[0].uids.len(), 1, "should drop the opt-out UID"); + assert_eq!(eids[0].uids[0].id, "id5-example", "should keep the UID"); + assert!( + matches!(eids[0].uids[0].expiry, ProviderExpiry::After(d) if d == crate::common::MIN_TTL), + "should keep a zero max age for the minimum TTL" + ); } #[test] @@ -413,9 +487,10 @@ mod tests { let provider = provider(); let headers = HeaderMap::new(); - for body in [&br#"{"universal_uid":"0"}"#[..], b"{}"] { + for body in [json!({ "universal_uid": "0" }), json!({})] { + let body = serde_json::to_vec(&body).expect("should encode body"); assert!( - matches!(respond(&*provider, &headers, body), LookupOutcome::NoMatch), + matches!(respond(&*provider, &headers, &body), LookupOutcome::NoMatch), "should report no match" ); } diff --git a/crates/trusted-server-identity-providers/src/liveintent.rs b/crates/trusted-server-identity-providers/src/liveintent.rs index c8f547bbf..275f89f66 100644 --- a/crates/trusted-server-identity-providers/src/liveintent.rs +++ b/crates/trusted-server-identity-providers/src/liveintent.rs @@ -8,17 +8,17 @@ use std::borrow::Cow; use std::collections::{BTreeMap, HashMap}; use std::num::NonZeroUsize; -use std::time::UNIX_EPOCH; +use std::time::{Duration, SystemTime}; use base64::Engine as _; use base64::engine::general_purpose::URL_SAFE_NO_PAD; use bytes::Bytes; use error_stack::{Report, ResultExt as _}; -use http::header::{EXPIRES, ORIGIN}; +use http::header::{DATE, EXPIRES, ORIGIN}; use http::{HeaderMap, HeaderValue, Method, StatusCode}; use serde::Deserialize; use serde_json::value::RawValue; -use trusted_server_core::ec::eid_types::{SourceDomain, UnixSeconds}; +use trusted_server_core::ec::eid_types::SourceDomain; use trusted_server_core::ec::provider::{ EidLevelFields, EidShape, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, ProviderConfigError, ProviderEid, ProviderError, ProviderExpiry, ProviderKind, @@ -26,7 +26,7 @@ use trusted_server_core::ec::provider::{ }; use url::Url; -use crate::common::{is_routable_ip, join_section_ids}; +use crate::common::{bounded_expiry, is_routable_ip, join_section_ids}; /// `atype` `LiveIntent` IDs carry: a person-based ID. const LIVEINTENT_ATYPE: i32 = 3; @@ -126,16 +126,22 @@ impl ProviderKind for LiveIntent { } } -/// Reads the `Expires` header as an absolute expiry. +/// Reads the `Expires` header as a bounded TTL. +/// +/// The TTL is `Expires` minus the response's `Date`, so it needs no local +/// clock. It is bounded by [`bounded_expiry`]: a missing or unparseable +/// `Expires` or `Date` gets the default TTL, so a UID never lives forever, +/// and an `Expires` at or before `Date` keeps the UID for the minimum TTL +/// instead of dropping a valid ID. fn expiry(headers: &HeaderMap) -> ProviderExpiry { - headers - .get(EXPIRES) - .and_then(|value| value.to_str().ok()) - .and_then(|value| httpdate::parse_http_date(value).ok()) - .and_then(|at| at.duration_since(UNIX_EPOCH).ok()) - .map_or(ProviderExpiry::Unspecified, |since| { - ProviderExpiry::At(UnixSeconds::new(since.as_secs())) - }) + let date = |name| -> Option { + let value = headers.get(name)?.to_str().ok()?; + httpdate::parse_http_date(value).ok() + }; + let ttl = date(EXPIRES) + .zip(date(DATE)) + .map(|(expires, date)| expires.duration_since(date).unwrap_or(Duration::ZERO)); + bounded_expiry(ttl) } impl IdentityProvider for LiveIntentProvider { @@ -203,7 +209,7 @@ impl IdentityProvider for LiveIntentProvider { return Err(Report::new(ProviderError::InvalidResponse) .attach(format!("LiveIntent returned status {}", response.status))); } - let reply: HashMap<&'body str, &'body RawValue> = + let reply: HashMap = serde_json::from_slice(response.body).change_context(ProviderError::InvalidResponse)?; let expiry = expiry(response.headers); let eids: Vec> = self @@ -300,7 +306,7 @@ mod tests { ec_id: "ec-example", consent: &consent, client: ClientSignals { - ip: Some("192.0.2.10".parse().expect("should parse ip")), + ip: Some("11.22.33.44".parse().expect("should parse ip")), user_agent: None, }, publisher_domain: "www.example.com", @@ -340,7 +346,7 @@ mod tests { ); assert_eq!( value("pip"), - [URL_SAFE_NO_PAD.encode("192.0.2.10").as_str()], + [URL_SAFE_NO_PAD.encode("11.22.33.44").as_str()], "should send the encoded IP" ); assert_eq!( @@ -356,15 +362,28 @@ mod tests { let mut headers = HeaderMap::new(); headers.insert( EXPIRES, + HeaderValue::from_static("Sun, 06 Nov 1994 10:49:37 GMT"), + ); + headers.insert( + DATE, HeaderValue::from_static("Sun, 06 Nov 1994 08:49:37 GMT"), ); - let body = br#"{"nonId":"li-example","partner":"partner-example","unknown":"x"}"#; + // Escape the `nonId` key as `LiveIntent` may, which a borrowed key + // cannot hold. + let body = json!({ + "nonId": "li-example", + "partner": "partner-example", + "unknown": "x", + }) + .to_string() + .replace("\"nonId\"", "\"non\\u0049d\"") + .into_bytes(); let outcome = provider .parse_response(ProviderResponse { status: StatusCode::OK, headers: &headers, - body, + body: &body, }) .expect("should parse response"); let LookupOutcome::Found(eids) = outcome else { @@ -376,8 +395,8 @@ mod tests { assert_eq!(eids[0].uids[0].id, "li-example", "should keep the ID"); assert_eq!(eids[0].uids[0].atype, Some(3), "should set atype 3"); assert!( - matches!(eids[0].uids[0].expiry, ProviderExpiry::At(at) if at.get() == 784_111_777), - "should use the Expires header" + matches!(eids[0].uids[0].expiry, ProviderExpiry::After(d) if d == Duration::from_secs(7200)), + "should use Expires relative to Date" ); assert_eq!( eids[1].source, "li-partner.example.com", @@ -385,14 +404,38 @@ mod tests { ); } + #[test] + fn bounds_a_missing_or_past_expires() { + let mut past = HeaderMap::new(); + past.insert( + EXPIRES, + HeaderValue::from_static("Sun, 06 Nov 1994 08:49:37 GMT"), + ); + past.insert( + DATE, + HeaderValue::from_static("Mon, 07 Nov 1994 08:49:37 GMT"), + ); + + for (headers, expected) in [ + (HeaderMap::new(), crate::common::DEFAULT_TTL), + (past, crate::common::MIN_TTL), + ] { + assert!( + matches!(expiry(&headers), ProviderExpiry::After(d) if d == expected), + "should bound the expiry to {expected:?}" + ); + } + } + #[test] fn reports_no_match_for_no_content_and_empty_ids() { let provider = provider(); let headers = HeaderMap::new(); + let empty_id = serde_json::to_vec(&json!({ "nonId": "" })).expect("should encode body"); for (status, body) in [ (StatusCode::NO_CONTENT, &b""[..]), - (StatusCode::OK, br#"{"nonId":""}"#), + (StatusCode::OK, &empty_id[..]), ] { let outcome = provider .parse_response(ProviderResponse { From 40a2239b6438e93d2817bd9b82dc1cfdece8e113 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 13:25:27 +0530 Subject: [PATCH 04/72] Let identity providers read their own first-party cookies Add an optional IdentityProvider::read_request_cookies step that runs for providers with due sources before any lookup. Cookie IDs pass the same consent gates, validation and write plan as lookup results and replace the lookup. The provider endpoint becomes optional so cookie-only kinds need none; lookup kinds reject a missing endpoint. Signed-off-by: dhruv8sh --- .../trusted-server-adapter-fastly/src/app.rs | 21 ++- .../trusted-server-adapter-fastly/src/main.rs | 3 +- .../src/ec/coordinator/mod.rs | 151 +++++++++++++++--- crates/trusted-server-core/src/ec/provider.rs | 110 +++++++++++-- crates/trusted-server-core/src/settings.rs | 6 +- .../src/id5.rs | 7 +- .../src/liveintent.rs | 8 +- 7 files changed, 267 insertions(+), 39 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index bce3fcc72..c811cc0de 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -122,7 +122,7 @@ use trusted_server_core::ec::eid_types::UnixSeconds; use trusted_server_core::ec::identify::{cors_preflight_identify, handle_identify}; use trusted_server_core::ec::kv::KvIdentityGraph; use trusted_server_core::ec::page_load::NavigationPermit; -use trusted_server_core::ec::provider::ClientSignals; +use trusted_server_core::ec::provider::{ClientSignals, RequestCookies}; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::ec::{EcContext, EcKvSnapshot, EidSyncSource}; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; @@ -395,6 +395,7 @@ pub(crate) struct AfterResponseLookups { pub(crate) ec_id: String, pub(crate) client_ip: Option, pub(crate) user_agent: Option, + pub(crate) cookie_header: String, } /// Identity work a navigation hands to the entry point. @@ -427,6 +428,7 @@ pub(crate) fn take_navigation_lookups() -> NavigationLookups { struct ProviderRequestSignals<'a> { client_ip: Option, user_agent: Option<&'a str>, + cookie_header: String, } impl<'a> ProviderRequestSignals<'a> { @@ -439,10 +441,23 @@ impl<'a> ProviderRequestSignals<'a> { .headers() .get(header::USER_AGENT) .and_then(|value| value.to_str().ok()), + cookie_header: joined_cookie_header(req), } } } +/// Joins every `Cookie` header on the request into one `; `-separated value. +/// +/// HTTP/2 and HTTP/3 clients may split cookies across several headers. +fn joined_cookie_header(req: &Request) -> String { + req.headers() + .get_all(header::COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .collect::>() + .join("; ") +} + /// Captures the request signals for after-response identity lookups on a /// permitted publisher navigation. /// @@ -461,6 +476,7 @@ fn prepare_after_response_lookups( ec_id, client_ip: signals.client_ip, user_agent: signals.user_agent.map(str::to_owned), + cookie_header: signals.cookie_header, }) } @@ -502,6 +518,7 @@ async fn dispatch_identity_lookups( user_agent: signals.user_agent, }, publisher_domain: &state.settings.publisher.domain, + cookies: RequestCookies::new(&signals.cookie_header), rate_limiter: &limiter, now: UnixSeconds::new(now), }, @@ -1630,7 +1647,7 @@ mod tests { use trusted_server_core::ec::provider::{ EidLevelFields, EidShape, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderEid, ProviderError, ProviderExpiry, ProviderRequest, - ProviderResponse, ProviderUid, SourceCapability, SourceInput, + ProviderResponse, ProviderUid, RequestCookies, SourceCapability, SourceInput, }; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::TrustedServerError; diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 6aafcc9c4..f2aff9de1 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -25,7 +25,7 @@ use trusted_server_core::ec::eid_types::UnixSeconds; use trusted_server_core::ec::finalize::ec_finalize_response; use trusted_server_core::ec::kv::KvIdentityGraph; use trusted_server_core::ec::page_load::NavigationPermit; -use trusted_server_core::ec::provider::ClientSignals; +use trusted_server_core::ec::provider::{ClientSignals, RequestCookies}; use trusted_server_core::ec::pull_sync::{ PullSyncContext, build_pull_sync_context, dispatch_pull_sync, }; @@ -801,6 +801,7 @@ pub(crate) fn run_after_response_lookups( user_agent: lookups.user_agent.as_deref(), }, publisher_domain: &settings.publisher.domain, + cookies: RequestCookies::new(&lookups.cookie_header), rate_limiter: &limiter, now: UnixSeconds::new(now), }, diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index c0cd83359..5641a0d5b 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -29,7 +29,7 @@ use crate::ec::generation::ec_hash; use crate::ec::kv_types::KvEntry; use crate::ec::provider::{ ClientSignals, IdentityProvider, LookupInput, LookupOutcome, OwnedEntries, ProviderConfigError, - ProviderEid, ProviderResponse, SourceInput, destination_allowed, parse_sources, + ProviderEid, ProviderResponse, RequestCookies, SourceInput, destination_allowed, parse_sources, validate_enabled_configs, }; use crate::ec::rate_limiter::RateLimiter; @@ -78,6 +78,8 @@ pub struct LookupContext<'a> { pub client: ClientSignals<'a>, /// Publisher domain. pub publisher_domain: &'a str, + /// Request cookies, read by providers that use first-party cookies. + pub cookies: RequestCookies<'a>, /// Existing EC rate limiter, used only for `rate_limit_per_hour`. pub rate_limiter: &'a dyn RateLimiter, /// Request time. @@ -102,6 +104,8 @@ pub struct PendingLookups { generated: Vec>, /// Provider state to hand back with each in-flight request's response. contexts: Vec>, + /// Providers whose IDs were read from first-party cookies, validated. + from_cookies: Vec>, } impl Default for PendingLookups { @@ -114,6 +118,7 @@ impl Default for PendingLookups { early: Vec::new(), generated: Vec::new(), contexts: Vec::new(), + from_cookies: Vec::new(), } } } @@ -124,6 +129,7 @@ impl std::fmt::Debug for PendingLookups { .field("requests", &self.requests.len()) .field("early", &self.early.len()) .field("generated", &self.generated.len()) + .field("from_cookies", &self.from_cookies.len()) .finish_non_exhaustive() } } @@ -132,7 +138,10 @@ impl PendingLookups { /// Whether nothing was dispatched, generated or skipped. #[must_use] pub fn is_empty(&self) -> bool { - self.requests.is_empty() && self.early.is_empty() && self.generated.is_empty() + self.requests.is_empty() + && self.early.is_empty() + && self.generated.is_empty() + && self.from_cookies.is_empty() } } @@ -521,6 +530,19 @@ impl IdentityCoordinator for ProviderCoordinator

{ })); continue; } + // IDs in the vendor's own first-party cookies stand in for a + // lookup. Only providers with due sources read them, so a held + // live UID is not rewritten on every navigation. + if let Some(eids) = registered.provider.read_request_cookies(lookup.cookies) { + let found = self.validated(provider, eids, lookup.now); + if matches!(found.status, LookupStatus::Found { .. }) { + pending.from_cookies.push(Tagged { + provider, + value: found, + }); + continue; + } + } let input = LookupInput { ec_id: lookup.ec_id, consent: lookup.consent, @@ -607,6 +629,7 @@ impl IdentityCoordinator for ProviderCoordinator

{ early, generated, contexts, + from_cookies, } = pending; let responses = match deadline { Some(deadline) if !requests.is_empty() => transport.gather(requests, deadline).await, @@ -617,7 +640,7 @@ impl IdentityCoordinator for ProviderCoordinator

{ for Tagged { provider, value } in early { slots[provider.get()] = Some(self.lookup(provider, value, Vec::new())); } - for Tagged { provider, value } in generated { + for Tagged { provider, value } in generated.into_iter().chain(from_cookies) { slots[provider.get()] = Some(value); } for Tagged { provider, value } in responses { @@ -672,7 +695,9 @@ mod tests { /// Context the echo provider sets on its request and expects back. const ECHO_CONTEXT: &str = "echo-context"; - struct EchoProvider(ProviderCapabilities); + /// Echoes the response body as one UID; when the cookie name is set, also + /// reads that cookie as one UID. + struct EchoProvider(ProviderCapabilities, Option<&'static str>); impl IdentityProvider for EchoProvider { fn capabilities(&self) -> &ProviderCapabilities { @@ -723,6 +748,26 @@ mod tests { }], }])) } + + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> Option>> { + let id = cookies.get(self.1?)?; + Some(vec![ProviderEid { + source: Cow::Borrowed("ids.example.com"), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids: vec![ProviderUid { + id: Cow::Borrowed(id), + atype: None, + ext: None, + expiry: ProviderExpiry::At(UnixSeconds::new(5000)), + }], + }]) + } } /// Answers every request with a canned result, recording the URLs sent. @@ -823,17 +868,24 @@ mod tests { } fn echo_provider(input: SourceInput) -> EchoProvider { - EchoProvider(ProviderCapabilities { - sources: Box::new([SourceCapability { - source: SourceDomain::try_from("ids.example.com").expect("should parse"), - input, - }]), - shape: EidShape { - max_eids: NonZeroUsize::new(4).expect("should be non-zero"), - max_uids_per_eid: NonZeroUsize::new(4).expect("should be non-zero"), - eid_level_fields: EidLevelFields::Preserved, + cookie_provider(input, None) + } + + fn cookie_provider(input: SourceInput, cookie: Option<&'static str>) -> EchoProvider { + EchoProvider( + ProviderCapabilities { + sources: Box::new([SourceCapability { + source: SourceDomain::try_from("ids.example.com").expect("should parse"), + input, + }]), + shape: EidShape { + max_eids: NonZeroUsize::new(4).expect("should be non-zero"), + max_uids_per_eid: NonZeroUsize::new(4).expect("should be non-zero"), + eid_level_fields: EidLevelFields::Preserved, + }, }, - }) + cookie, + ) } fn coordinator_with( @@ -890,6 +942,7 @@ mod tests { user_agent: None, }, publisher_domain: "www.example.com", + cookies: RequestCookies::new("vendor_id=c-1"), rate_limiter: limiter, now: UnixSeconds::new(1000), }; @@ -1286,6 +1339,63 @@ mod tests { } } + #[test] + fn first_party_cookie_id_replaces_the_lookup() { + let coordinator = coordinator_with(RegisteredProvider::new( + provider_config(""), + cookie_provider(SourceInput::Anonymous, Some("vendor_id")), + )); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + let lookup = results.iter().next().expect("should report the provider"); + assert!( + matches!(lookup.status, LookupStatus::Found { kept: 1, .. }), + "should keep the cookie UID" + ); + assert_eq!( + lookup.entries[0].1[0].uids()[0].id().as_str(), + "c-1", + "should store the cookie value" + ); + assert!(transport.sent.borrow().is_empty(), "should send no lookup"); + } + + #[test] + fn missing_or_invalid_cookie_falls_back_to_the_lookup() { + let coordinator = coordinator_with(RegisteredProvider::new( + provider_config(""), + cookie_provider(SourceInput::Anonymous, Some("absent")), + )); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + assert!( + matches!(only_status(&results), LookupStatus::Found { kept: 1, .. }), + "should use the lookup" + ); + assert_eq!(transport.sent.borrow().len(), 1, "should send the lookup"); + } + + #[test] + fn cookie_is_not_read_without_consent() { + let coordinator = coordinator_with(RegisteredProvider::new( + provider_config("tcf_vendor_id = 9999"), + cookie_provider(SourceInput::Anonymous, Some("vendor_id")), + )); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::Consent), + "should skip for consent" + ); + } + #[test] fn disabled_provider_is_not_registered() { let coordinator = coordinator("enabled = false"); @@ -1296,7 +1406,7 @@ mod tests { #[test] fn endpoint_outside_allowlist_is_rejected_at_build() { let mut config = provider_config(""); - config.endpoint = "https://other.example.com/".to_owned(); + config.endpoint = Some("https://other.example.com/".to_owned()); let error = ProviderCoordinator::new( vec![RegisteredProvider::new( @@ -1317,10 +1427,13 @@ mod tests { #[test] fn source_outside_capabilities_is_rejected_at_build() { - let provider = EchoProvider(ProviderCapabilities { - sources: Box::new([]), - ..echo_provider(SourceInput::Anonymous).0 - }); + let provider = EchoProvider( + ProviderCapabilities { + sources: Box::new([]), + ..echo_provider(SourceInput::Anonymous).0 + }, + None, + ); let error = ProviderCoordinator::new( vec![RegisteredProvider::new(provider_config(""), provider)], diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs index 9f479fa29..eb3373071 100644 --- a/crates/trusted-server-core/src/ec/provider.rs +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -70,6 +70,20 @@ pub trait IdentityProvider: Send + Sync { let _ = lookup; None } + + /// Reads IDs from the vendor's own first-party cookies on the request. + /// + /// Runs before any lookup for providers with due sources. Returned + /// entries go through the same validation, consent gates and write plan + /// as lookup results, and a kept UID makes the provider skip its lookup + /// for this navigation. The default reads nothing. + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> Option>> { + let _ = cookies; + None + } } impl IdentityProvider for Box

{ @@ -94,6 +108,39 @@ impl IdentityProvider for Box

{ fn generate(&self, lookup: &LookupInput<'_>) -> Option>> { (**self).generate(lookup) } + + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> Option>> { + (**self).read_request_cookies(cookies) + } +} + +/// The request's `Cookie` header, lent to providers that read their own +/// first-party cookies. +#[derive(Debug, Clone, Copy, Default)] +pub struct RequestCookies<'a>(&'a str); + +impl<'a> RequestCookies<'a> { + /// Wraps a `Cookie` header value (pairs separated by `;`). + #[must_use] + pub fn new(header: &'a str) -> Self { + Self(header) + } + + /// Returns the first value of cookie `name`, undecoded and unquoted. + /// + /// An empty value reads as absent. + #[must_use] + pub fn get(&self, name: &str) -> Option<&'a str> { + self.0 + .split(';') + .filter_map(|pair| pair.split_once('=')) + .find(|(key, _)| key.trim() == name) + .map(|(_, value)| value.trim().trim_matches('"')) + .filter(|value| !value.is_empty()) + } } /// What a provider can return, used by the coordinator to apply shared policy. @@ -404,8 +451,8 @@ pub trait ProviderKind { /// Shared configuration every provider receives at build time. #[derive(Debug, Clone, Copy)] pub struct ProviderCommon<'a> { - /// Configured lookup endpoint. - pub endpoint: &'a Url, + /// Configured lookup endpoint, absent for cookie-only kinds. + pub endpoint: Option<&'a Url>, /// Configured sources. pub sources: &'a [SourceDomain], /// Resolved secrets. @@ -439,12 +486,10 @@ pub fn build_kind( let settings = K::Settings::deserialize(serde_json::Value::Object(config.settings.clone())) .change_context(ProviderConfigError::Invalid) .attach_with(|| format!("settings for provider `{}` are invalid", config.id))?; - let endpoint = Url::parse(&config.endpoint) - .change_context(ProviderConfigError::Invalid) - .attach_with(|| format!("endpoint for provider `{}` is not a URL", config.id))?; + let endpoint = parse_endpoint(config)?; let sources = parse_sources(config)?; let common = ProviderCommon { - endpoint: &endpoint, + endpoint: endpoint.as_ref(), sources: &sources, secrets, }; @@ -452,6 +497,21 @@ pub fn build_kind( Ok(Box::new(provider)) } +/// Parses a provider's configured endpoint, when one is set. +fn parse_endpoint( + config: &IdentityProviderConfig, +) -> Result, Report> { + config + .endpoint + .as_deref() + .map(|endpoint| { + Url::parse(endpoint) + .change_context(ProviderConfigError::Invalid) + .attach_with(|| format!("endpoint for provider `{}` is not a URL", config.id)) + }) + .transpose() +} + /// Parses and normalizes a provider's configured sources. pub(crate) fn parse_sources( config: &IdentityProviderConfig, @@ -478,7 +538,7 @@ pub(crate) fn destination_allowed>(url: &Url, allowed_domains: &[S } /// Checks the configuration rules core owns for every enabled provider -/// (spec §12.1): the endpoint is an `https` URL whose host is in +/// (spec §12.1): a configured endpoint is an `https` URL whose host is in /// `allowed_domains`, it claims at least one source, every source has an /// `[[ec.partners]]` entry, no source is claimed by two enabled providers, and /// every enabled provider has a non-empty id of its own. @@ -507,10 +567,9 @@ pub fn validate_enabled_configs<'a>( config.id ))); } - let endpoint = Url::parse(&config.endpoint) - .change_context(ProviderConfigError::Invalid) - .attach_with(|| format!("endpoint for provider `{}` is not a URL", config.id))?; - if !destination_allowed(&endpoint, &config.allowed_domains) { + let endpoint = parse_endpoint(config)?; + if endpoint.is_some_and(|endpoint| !destination_allowed(&endpoint, &config.allowed_domains)) + { return Err(Report::new(ProviderConfigError::Invalid).attach(format!( "endpoint for provider `{}` must be https with a host in allowed_domains", config.id @@ -758,9 +817,9 @@ mod tests { #[test] fn validate_enabled_configs_rejects_endpoint_outside_allowlist() { let mut outside = config(&json!({})); - outside.endpoint = "https://other.example.com/".to_owned(); + outside.endpoint = Some("https://other.example.com/".to_owned()); let mut plain_http = config(&json!({})); - plain_http.endpoint = "http://ids.example.com/".to_owned(); + plain_http.endpoint = Some("http://ids.example.com/".to_owned()); for rejected in [outside, plain_http] { let error = validate_enabled_configs([&rejected], &partners()) @@ -856,12 +915,35 @@ mod tests { let first = config(&json!({})); let mut disabled = config(&json!({})); disabled.enabled = false; - disabled.endpoint = "https://other.example.com/".to_owned(); + disabled.endpoint = Some("https://other.example.com/".to_owned()); validate_enabled_configs([&first, &disabled], &partners()) .expect("should ignore disabled provider"); } + #[test] + fn validate_enabled_configs_accepts_cookie_only_provider_without_endpoint() { + let mut cookie_only = config(&json!({})); + cookie_only.endpoint = None; + + validate_enabled_configs([&cookie_only], &partners()) + .expect("should accept a provider without endpoint"); + } + + #[test] + fn request_cookies_finds_named_values() { + let cookies = RequestCookies::new("a=1; vendor_id=\"v-1\" ;empty=; vendor_id=v-2"); + + assert_eq!( + cookies.get("vendor_id"), + Some("v-1"), + "should find first value" + ); + assert_eq!(cookies.get("a"), Some("1"), "should find leading cookie"); + assert_eq!(cookies.get("empty"), None, "should ignore empty values"); + assert_eq!(cookies.get("missing"), None, "should miss absent cookies"); + } + #[test] fn owned_entries_yields_only_configured_sources() { let entry: KvEntry = serde_json::from_value(json!({ diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 33544d77d..b528bffca 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -651,7 +651,11 @@ pub struct IdentityProviderConfig { )] pub enabled: bool, /// Lookup endpoint; must be `https` with a host in `allowed_domains`. - pub endpoint: String, + /// + /// Optional for kinds that only read first-party cookies; kinds that + /// make lookups reject a missing endpoint when they are built. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub endpoint: Option, /// Exact hosts the coordinator may call for this provider. #[serde(default, deserialize_with = "vec_from_seq_or_map")] pub allowed_domains: Vec, diff --git a/crates/trusted-server-identity-providers/src/id5.rs b/crates/trusted-server-identity-providers/src/id5.rs index 036d6eb69..ab1f5add9 100644 --- a/crates/trusted-server-identity-providers/src/id5.rs +++ b/crates/trusted-server-identity-providers/src/id5.rs @@ -78,7 +78,12 @@ impl ProviderKind for Id5 { return Err(Report::new(ProviderConfigError::Invalid) .attach("an id5 provider needs at least one source")); } - let mut url = common.endpoint.clone(); + let mut url = common + .endpoint + .ok_or_else(|| { + Report::new(ProviderConfigError::Invalid).attach("a id5 provider needs an endpoint") + })? + .clone(); url.path_segments_mut() .map_err(|()| { Report::new(ProviderConfigError::Invalid).attach("endpoint cannot hold a path") diff --git a/crates/trusted-server-identity-providers/src/liveintent.rs b/crates/trusted-server-identity-providers/src/liveintent.rs index 275f89f66..b61ea9226 100644 --- a/crates/trusted-server-identity-providers/src/liveintent.rs +++ b/crates/trusted-server-identity-providers/src/liveintent.rs @@ -96,7 +96,13 @@ impl ProviderKind for LiveIntent { .attach("a liveintent provider needs at least one attribute") })?; let distributor = settings.distributor_id.as_deref().unwrap_or("prebid"); - let mut url = common.endpoint.clone(); + let mut url = common + .endpoint + .ok_or_else(|| { + Report::new(ProviderConfigError::Invalid) + .attach("a liveintent provider needs an endpoint") + })? + .clone(); url.path_segments_mut() .map_err(|()| { Report::new(ProviderConfigError::Invalid).attach("endpoint cannot hold a path") From 132f33c3c405cdfcdbfdd18145cbcb8c939773d4 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 13:27:13 +0530 Subject: [PATCH 05/72] Add the LiveRamp identity provider Signed-off-by: dhruv8sh --- Cargo.lock | 1 + .../Cargo.toml | 1 + .../src/common.rs | 96 +++++- .../src/lib.rs | 7 +- .../src/liveramp.rs | 281 ++++++++++++++++++ .../2026-11-02-identity-provider-liveramp.md | 33 ++ trusted-server.example.toml | 13 + 7 files changed, 430 insertions(+), 2 deletions(-) create mode 100644 crates/trusted-server-identity-providers/src/liveramp.rs create mode 100644 docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md diff --git a/Cargo.lock b/Cargo.lock index 4d558019e..554040b1f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6753,6 +6753,7 @@ dependencies = [ "toml", "trusted-server-core", "url", + "urlencoding", ] [[package]] diff --git a/crates/trusted-server-identity-providers/Cargo.toml b/crates/trusted-server-identity-providers/Cargo.toml index e01ab9378..5cfe89243 100644 --- a/crates/trusted-server-identity-providers/Cargo.toml +++ b/crates/trusted-server-identity-providers/Cargo.toml @@ -24,6 +24,7 @@ serde = { workspace = true } serde_json = { workspace = true, features = ["raw_value"] } trusted-server-core = { workspace = true } url = { workspace = true } +urlencoding = { workspace = true } [dev-dependencies] toml = { workspace = true } diff --git a/crates/trusted-server-identity-providers/src/common.rs b/crates/trusted-server-identity-providers/src/common.rs index 5d37e39fb..242a0ba17 100644 --- a/crates/trusted-server-identity-providers/src/common.rs +++ b/crates/trusted-server-identity-providers/src/common.rs @@ -2,11 +2,17 @@ use std::borrow::Cow; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::num::NonZeroUsize; use std::time::Duration; +use error_stack::Report; use serde::Deserialize; use serde_json::value::RawValue; -use trusted_server_core::ec::provider::{ProviderEid, ProviderExpiry, ProviderUid}; +use trusted_server_core::ec::eid_types::SourceDomain; +use trusted_server_core::ec::provider::{ + EidLevelFields, EidShape, ProviderCapabilities, ProviderCommon, ProviderConfigError, + ProviderEid, ProviderExpiry, ProviderUid, SourceCapability, SourceInput, +}; /// Returns whether `ip` is a public address worth sending to a vendor. /// @@ -109,6 +115,76 @@ pub(crate) fn join_section_ids(ids: &[u16]) -> String { ids.iter().map(u16::to_string).collect::>().join(",") } +/// Agent type of the person-based IDs read from vendor cookies. +pub(crate) const COOKIE_ATYPE: i32 = 3; + +/// Builds the capabilities of a cookie-only provider, which claims exactly +/// one source and stores one UID for it. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError::Invalid`] unless exactly one source is +/// configured. +pub(crate) fn cookie_capabilities( + common: &ProviderCommon<'_>, + kind: &str, +) -> Result<(ProviderCapabilities, SourceDomain), Report> { + let [source] = common.sources else { + return Err(Report::new(ProviderConfigError::Invalid) + .attach(format!("a {kind} provider needs exactly one source"))); + }; + let one = NonZeroUsize::MIN; + let capabilities = ProviderCapabilities { + sources: Box::new([SourceCapability { + source: source.clone(), + input: SourceInput::Anonymous, + }]), + shape: EidShape { + max_eids: one, + max_uids_per_eid: one, + eid_level_fields: EidLevelFields::Ignored, + }, + }; + Ok((capabilities, source.clone())) +} + +/// Whether a cookie-carried ID is worth storing: not blank and not an +/// opt-out placeholder such as `0` or an all-zero UUID. +#[must_use] +pub(crate) fn is_usable_id(id: &str) -> bool { + let id = id.trim(); + !id.is_empty() && !id.chars().all(|c| c == '0' || c == '-') +} + +/// One EID for `source` holding the single UID `id` with agent type 3. +#[must_use] +pub(crate) fn cookie_eid<'cookie>( + source: &SourceDomain, + id: Cow<'cookie, str>, + expiry: ProviderExpiry, +) -> ProviderEid<'cookie> { + ProviderEid { + source: Cow::Owned(source.as_str().to_owned()), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids: vec![ProviderUid { + id, + atype: Some(COOKIE_ATYPE), + ext: None, + expiry, + }], + } +} + +/// Percent-decodes a cookie value, keeping it unchanged when it is not +/// valid percent-encoded UTF-8. +#[must_use] +pub(crate) fn percent_decoded(value: &str) -> Cow<'_, str> { + urlencoding::decode(value).unwrap_or(Cow::Borrowed(value)) +} + /// A complete `OpenRTB` EID as a vendor returns it, borrowed from the body. /// /// Unknown fields are ignored so one new vendor field does not fail the @@ -227,6 +303,24 @@ mod tests { } } + #[test] + fn filters_blank_and_opt_out_ids() { + for id in ["", " ", "0", "00000000-0000-0000-0000-000000000000"] { + assert!(!is_usable_id(id), "should drop {id:?}"); + } + assert!(is_usable_id("id-example-1"), "should keep a real ID"); + } + + #[test] + fn percent_decodes_leniently() { + assert_eq!(percent_decoded("a%20b"), "a b", "should decode"); + assert_eq!( + percent_decoded("a+b%zz"), + "a+b%zz", + "should keep invalid input" + ); + } + #[test] fn bounds_vendor_expiry() { for (ttl, expected) in [ diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index f1ea82c3b..70eed9fe6 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -12,6 +12,7 @@ mod common; pub mod id5; pub mod liveintent; +pub mod liveramp; use std::sync::Arc; use std::time::Duration; @@ -31,7 +32,10 @@ use trusted_server_core::settings::{ /// #[must_use] pub fn is_registered_kind(kind: &str) -> bool { - matches!(kind, id5::Id5::KIND | liveintent::LiveIntent::KIND) + matches!( + kind, + id5::Id5::KIND | liveintent::LiveIntent::KIND | liveramp::LiveRamp::KIND + ) } /// Builds the partner registry, marking the sources claimed by enabled @@ -58,6 +62,7 @@ pub fn build( match config.kind() { id5::Id5::KIND => build_kind::(config, secrets), liveintent::LiveIntent::KIND => build_kind::(config, secrets), + liveramp::LiveRamp::KIND => build_kind::(config, secrets), kind => Err(Report::new(ProviderConfigError::UnknownKind) .attach(format!("kind `{kind}` is not registered"))), } diff --git a/crates/trusted-server-identity-providers/src/liveramp.rs b/crates/trusted-server-identity-providers/src/liveramp.rs new file mode 100644 index 000000000..3a091ca3e --- /dev/null +++ b/crates/trusted-server-identity-providers/src/liveramp.rs @@ -0,0 +1,281 @@ +//! `LiveRamp` identity provider: reads the `RampID` envelope from `LiveRamp`'s own +//! first-party cookie. +//! +//! `LiveRamp`'s ATS script stores the envelope in the `_lr_env` cookie as +//! base64 (or base64url) of a JSON object whose `envelope` field holds it. +//! The envelope is stored unchanged with agent type 3 for +//! [`ENVELOPE_TTL`]. No server call is made: the refresh contract is not +//! confirmed, so the browser keeps refreshing the cookie. +//! +//! Lockr AIM can write the same envelope into a cookie whose name comes +//! from its own configuration; list that name in `cookie_names`. +//! +//! Parsing is lenient: an undecodable cookie, a body without a string +//! `envelope`, or a blank or opt-out envelope reads as no ID. + +use std::borrow::Cow; +use std::time::Duration; + +use base64::Engine as _; +use base64::engine::general_purpose::{STANDARD, STANDARD_NO_PAD, URL_SAFE, URL_SAFE_NO_PAD}; +use error_stack::Report; +use serde::Deserialize; +use trusted_server_core::ec::eid_types::SourceDomain; +use trusted_server_core::ec::provider::{ + IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, + ProviderConfigError, ProviderEid, ProviderError, ProviderKind, ProviderRequest, + ProviderResponse, RequestCookies, +}; + +use crate::common::{ + bounded_expiry, cookie_capabilities, cookie_eid, is_usable_id, percent_decoded, +}; + +/// How long a stored envelope is kept. +pub const ENVELOPE_TTL: Duration = Duration::from_secs(15 * 24 * 60 * 60); + +/// The `LiveRamp` provider kind. +pub struct LiveRamp; + +/// Settings for a `liveramp` provider. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LiveRampSettings { + /// Cookies read in order; the first holding an envelope wins. + #[serde(default = "LiveRampSettings::default_cookie_names")] + pub cookie_names: Vec, +} + +impl LiveRampSettings { + fn default_cookie_names() -> Vec { + vec!["_lr_env".to_owned()] + } +} + +/// A built `LiveRamp` provider. +#[derive(Debug)] +pub struct LiveRampProvider { + capabilities: ProviderCapabilities, + source: SourceDomain, + cookie_names: Box<[String]>, +} + +impl ProviderKind for LiveRamp { + const KIND: &'static str = "liveramp"; + type Settings = LiveRampSettings; + type Provider = LiveRampProvider; + + fn build( + common: &ProviderCommon<'_>, + settings: LiveRampSettings, + ) -> Result> { + if settings.cookie_names.is_empty() { + return Err(Report::new(ProviderConfigError::Invalid) + .attach("a liveramp provider needs at least one cookie name")); + } + let (capabilities, source) = cookie_capabilities(common, Self::KIND)?; + Ok(LiveRampProvider { + capabilities, + source, + cookie_names: settings.cookie_names.into_boxed_slice(), + }) + } +} + +#[derive(Deserialize)] +struct EnvelopeCookie { + envelope: String, +} + +/// Decodes one `_lr_env` value into its envelope. +fn envelope(value: &str) -> Option { + let value = percent_decoded(value); + let value = value.trim(); + let bytes = [STANDARD, STANDARD_NO_PAD, URL_SAFE, URL_SAFE_NO_PAD] + .iter() + .find_map(|engine| engine.decode(value).ok())?; + let cookie: EnvelopeCookie = serde_json::from_slice(&bytes).ok()?; + is_usable_id(&cookie.envelope).then_some(cookie.envelope) +} + +impl IdentityProvider for LiveRampProvider { + fn capabilities(&self) -> &ProviderCapabilities { + &self.capabilities + } + + fn build_request( + &self, + _lookup: &LookupInput<'_>, + ) -> Result> { + Err(Report::new(ProviderError::Request).attach("liveramp makes no lookup")) + } + + fn parse_response<'body>( + &self, + _response: ProviderResponse<'body>, + ) -> Result, Report> { + Err(Report::new(ProviderError::InvalidResponse).attach("liveramp makes no lookup")) + } + + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> Option>> { + let envelope = self + .cookie_names + .iter() + .filter_map(|name| cookies.get(name)) + .find_map(envelope)?; + Some(vec![cookie_eid( + &self.source, + Cow::Owned(envelope), + bounded_expiry(Some(ENVELOPE_TTL)), + )]) + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + use trusted_server_core::consent::types::ConsentContext; + use trusted_server_core::ec::provider::{ + ClientSignals, OwnedEntries, ProviderExpiry, build_kind, + }; + use trusted_server_core::settings::{IdentityProviderConfig, ProviderSecrets}; + + use super::*; + + fn provider(settings: &serde_json::Value) -> Box { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "liveramp", + "kind": "liveramp", + "sources": ["liveramp.example.com"], + "settings": settings, + })) + .expect("should parse provider config"); + build_kind::(&config, &ProviderSecrets::default()).expect("should build provider") + } + + fn encoded(body: &serde_json::Value, engine: &base64::engine::GeneralPurpose) -> String { + engine.encode(body.to_string()) + } + + fn read(provider: &dyn IdentityProvider, header: &str) -> Option<(String, ProviderExpiry)> { + let eids = provider.read_request_cookies(RequestCookies::new(header))?; + assert_eq!(eids.len(), 1, "should return one EID"); + assert_eq!( + eids[0].source, "liveramp.example.com", + "should use the source" + ); + let uid = &eids[0].uids[0]; + assert_eq!(uid.atype, Some(3), "should be person-based"); + Some((uid.id.to_string(), uid.expiry)) + } + + #[test] + fn reads_the_envelope_in_base64_and_base64url() { + let provider = provider(&json!({})); + let body = json!({ "envelope": "AjfowMv4ZHZQJFM8TpiUnYEyA81Vdgg", "ts": 1 }); + + for engine in [STANDARD, URL_SAFE_NO_PAD] { + let (id, expiry) = read(&*provider, &format!("_lr_env={}", encoded(&body, &engine))) + .expect("should read the envelope"); + + assert_eq!( + id, "AjfowMv4ZHZQJFM8TpiUnYEyA81Vdgg", + "should keep the envelope" + ); + assert!( + matches!(expiry, ProviderExpiry::After(after) if after == ENVELOPE_TTL), + "should keep for 15 days" + ); + } + } + + #[test] + fn reads_a_configured_cookie_name() { + let provider = provider(&json!({ "cookie_names": ["example_lockr_env"] })); + let value = encoded(&json!({ "envelope": "env-example" }), &STANDARD); + + let read = read( + &*provider, + &format!("_lr_env=ignored; example_lockr_env={value}"), + ); + + assert_eq!( + read.map(|(id, _)| id).as_deref(), + Some("env-example"), + "should read it" + ); + } + + #[test] + fn ignores_missing_malformed_and_opt_out_cookies() { + let provider = provider(&json!({})); + for header in [ + String::new(), + "_lr_env=not-base64!".to_owned(), + format!("_lr_env={}", encoded(&json!({ "other": "x" }), &STANDARD)), + format!("_lr_env={}", encoded(&json!({ "envelope": 5 }), &STANDARD)), + format!("_lr_env={}", encoded(&json!({ "envelope": "" }), &STANDARD)), + format!( + "_lr_env={}", + encoded(&json!({ "envelope": "0" }), &STANDARD) + ), + ] { + assert!( + read(&*provider, &header).is_none(), + "should ignore {header:?}" + ); + } + } + + #[test] + fn requires_exactly_one_source_and_a_cookie_name() { + for (sources, settings) in [ + (json!([]), json!({})), + (json!(["a.example.com", "b.example.com"]), json!({})), + (json!(["a.example.com"]), json!({ "cookie_names": [] })), + ] { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "liveramp", + "kind": "liveramp", + "sources": sources, + "settings": settings, + })) + .expect("should parse provider config"); + + let error = build_kind::(&config, &ProviderSecrets::default()) + .err() + .expect("should reject config"); + + assert!( + matches!(error.current_context(), ProviderConfigError::Invalid), + "should report invalid config" + ); + } + } + + #[test] + fn makes_no_lookup() { + let provider = provider(&json!({})); + let consent = ConsentContext::default(); + let lookup = LookupInput { + ec_id: "ec-example", + consent: &consent, + client: ClientSignals { + ip: None, + user_agent: None, + }, + publisher_domain: "www.example.com", + due_sources: &[], + current: OwnedEntries::new(None, &[]), + hashed_email: None, + }; + + assert!( + provider.build_request(&lookup).is_err(), + "should build no request" + ); + } +} diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md b/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md new file mode 100644 index 000000000..c91ce6c50 --- /dev/null +++ b/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md @@ -0,0 +1,33 @@ +# LiveRamp Identity Provider Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Store the LiveRamp RampID envelope the browser already sends in LiveRamp's own first-party cookie, as an `[[ec.identity_providers]]` entry with `kind = "liveramp"`. No login and no server call. + +**Architecture:** `crates/trusted-server-identity-providers/src/liveramp.rs` holds `LiveRamp` (a `ProviderKind`), `LiveRampSettings` and `LiveRampProvider`. The provider implements only `IdentityProvider::read_request_cookies`; `build_request` always fails, so the coordinator never sends a lookup. The coordinator reads cookies only for due sources and applies consent, validation, expiry and persistence exactly as for lookup results. + +**Spec:** `docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md`, section 7. + +## Cookie + +- Default cookie `_lr_env`; `cookie_names` overrides the list, read in order. +- The value is base64 or base64url (padded or not, optionally percent-encoded) of a JSON object. Only its string `envelope` field is used, unchanged. +- A blank or opt-out envelope (`0`, all zeros) and any undecodable value read as no ID. + +## Stored ID + +- One EID for the single configured source, one UID with `atype = 3`. +- Expiry `ProviderExpiry::After(15 days)` through `bounded_expiry`. +- No refresh call: LiveRamp's server-side refresh contract is not confirmed, so the browser keeps refreshing the cookie. Revisit when it is. + +## Lockr + +Lockr AIM writes the LiveRamp envelope into a cookie named by its own configuration. List that name in `cookie_names`; no separate Lockr kind is needed. + +## Tasks + +- [ ] Add `cookie_capabilities`, `cookie_eid`, `is_usable_id` and `percent_decoded` to `common.rs`. +- [ ] Add `liveramp.rs` and register the kind in `is_registered_kind` and `build`. +- [ ] Unit tests: base64 and base64url envelopes, configured cookie name, missing, malformed and opt-out cookies ignored, one-source and cookie-name config checks, no lookup built. +- [ ] Add a commented `kind = "liveramp"` example to `trusted-server.example.toml`. +- [ ] Run `cargo test-fastly` and `cargo clippy-fastly`. diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 66f2826f2..3f13c0649 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -180,6 +180,19 @@ pull_sync_concurrency = 3 # distributor_id = "example-distributor" # [ec.identity_providers.settings.attributes] # nonId = "liveintent.example.com" +# +# LiveRamp (kind "liveramp"): no lookup and no endpoint. Reads the RampID +# envelope from LiveRamp's own first-party `_lr_env` cookie and keeps it for +# 15 days. Takes exactly one source; use the one LiveRamp gives you. Add a +# cookie name that Lockr AIM writes the envelope into to `cookie_names`. +# [[ec.identity_providers]] +# id = "liveramp" +# kind = "liveramp" +# enabled = false +# sources = ["liveramp.example.com"] +# tcf_vendor_id = 9999 +# [ec.identity_providers.settings] +# cookie_names = ["_lr_env"] # ============================================================================= From 90c5cbeea12d9453a9cb526926d932820c03212b Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 13:27:58 +0530 Subject: [PATCH 06/72] Add the Yahoo ConnectID identity provider Signed-off-by: dhruv8sh --- .../src/connectid.rs | 234 ++++++++++++++++++ .../src/lib.rs | 7 +- .../2026-11-02-identity-provider-connectid.md | 28 +++ trusted-server.example.toml | 10 + 4 files changed, 278 insertions(+), 1 deletion(-) create mode 100644 crates/trusted-server-identity-providers/src/connectid.rs create mode 100644 docs/superpowers/plans/2026-11-02-identity-provider-connectid.md diff --git a/crates/trusted-server-identity-providers/src/connectid.rs b/crates/trusted-server-identity-providers/src/connectid.rs new file mode 100644 index 000000000..aaddfeffd --- /dev/null +++ b/crates/trusted-server-identity-providers/src/connectid.rs @@ -0,0 +1,234 @@ +//! Yahoo `ConnectID` identity provider: reads `ConnectID` from Yahoo's own +//! first-party cookie. +//! +//! Yahoo's `ConnectID` module stores a JSON object in the `connectId` cookie +//! (optionally percent-encoded) with `connectId`, `lastSynced` and `ttl`, +//! both in milliseconds. The ID is stored with agent type 3 until +//! `lastSynced + ttl`, with the TTL capped at [`MAX_TTL`]; the coordinator +//! drops it once that time has passed. A missing `ttl` uses the shared +//! default TTL. No server call is made. +//! +//! The cookie's `he` and `puid` fields are never read, so they are never +//! stored. Parsing is lenient: an undecodable cookie, a missing +//! `connectId` or `lastSynced`, or a blank or opt-out ID reads as no ID. + +use std::borrow::Cow; +use std::time::Duration; + +use error_stack::Report; +use serde::Deserialize; +use trusted_server_core::ec::eid_types::{SourceDomain, UnixSeconds}; +use trusted_server_core::ec::provider::{ + IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, + ProviderConfigError, ProviderEid, ProviderError, ProviderExpiry, ProviderKind, ProviderRequest, + ProviderResponse, RequestCookies, +}; + +use crate::common::{DEFAULT_TTL, cookie_capabilities, cookie_eid, is_usable_id, percent_decoded}; + +/// Longest time a `ConnectID` is kept after its last sync. +pub const MAX_TTL: Duration = Duration::from_secs(14 * 24 * 60 * 60); + +/// The Yahoo `ConnectID` provider kind. +pub struct ConnectId; + +/// Settings for a `connectid` provider. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ConnectIdSettings { + /// Cookie holding the `ConnectID` object. + #[serde(default = "ConnectIdSettings::default_cookie_name")] + pub cookie_name: String, +} + +impl ConnectIdSettings { + fn default_cookie_name() -> String { + "connectId".to_owned() + } +} + +/// A built Yahoo `ConnectID` provider. +#[derive(Debug)] +pub struct ConnectIdProvider { + capabilities: ProviderCapabilities, + source: SourceDomain, + cookie_name: String, +} + +impl ProviderKind for ConnectId { + const KIND: &'static str = "connectid"; + type Settings = ConnectIdSettings; + type Provider = ConnectIdProvider; + + fn build( + common: &ProviderCommon<'_>, + settings: ConnectIdSettings, + ) -> Result> { + let (capabilities, source) = cookie_capabilities(common, Self::KIND)?; + Ok(ConnectIdProvider { + capabilities, + source, + cookie_name: settings.cookie_name, + }) + } +} + +/// The fields of the `connectId` cookie this provider reads. +#[derive(Deserialize)] +struct ConnectIdCookie { + #[serde(rename = "connectId")] + connect_id: String, + #[serde(rename = "lastSynced")] + last_synced_ms: f64, + #[serde(default)] + ttl: Option, +} + +/// Milliseconds as a duration; negative or non-finite values are zero. +fn from_millis(ms: f64) -> Duration { + Duration::try_from_secs_f64(ms / 1000.0).unwrap_or(Duration::ZERO) +} + +/// Decodes one cookie value into the ID and its expiry. +fn connect_id(value: &str) -> Option<(String, ProviderExpiry)> { + let decoded = percent_decoded(value); + let cookie: ConnectIdCookie = serde_json::from_str(&decoded).ok()?; + if !is_usable_id(&cookie.connect_id) { + return None; + } + let ttl = cookie.ttl.map_or(DEFAULT_TTL, from_millis).min(MAX_TTL); + let expires = from_millis(cookie.last_synced_ms).saturating_add(ttl); + Some(( + cookie.connect_id, + ProviderExpiry::At(UnixSeconds::new(expires.as_secs())), + )) +} + +impl IdentityProvider for ConnectIdProvider { + fn capabilities(&self) -> &ProviderCapabilities { + &self.capabilities + } + + fn build_request( + &self, + _lookup: &LookupInput<'_>, + ) -> Result> { + Err(Report::new(ProviderError::Request).attach("connectid makes no lookup")) + } + + fn parse_response<'body>( + &self, + _response: ProviderResponse<'body>, + ) -> Result, Report> { + Err(Report::new(ProviderError::InvalidResponse).attach("connectid makes no lookup")) + } + + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> Option>> { + let (id, expiry) = connect_id(cookies.get(&self.cookie_name)?)?; + Some(vec![cookie_eid(&self.source, Cow::Owned(id), expiry)]) + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + use trusted_server_core::ec::provider::build_kind; + use trusted_server_core::settings::{IdentityProviderConfig, ProviderSecrets}; + + use super::*; + + fn provider() -> Box { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "connectid", + "kind": "connectid", + "sources": ["connectid.example.com"], + })) + .expect("should parse provider config"); + build_kind::(&config, &ProviderSecrets::default()) + .expect("should build provider") + } + + fn read(cookie: &serde_json::Value) -> Option<(String, ProviderExpiry)> { + let header = format!("connectId={}", urlencoding::encode(&cookie.to_string())); + let eids = provider().read_request_cookies(RequestCookies::new(&header))?; + assert_eq!( + eids[0].source, "connectid.example.com", + "should use the source" + ); + let uid = &eids[0].uids[0]; + assert_eq!(uid.atype, Some(3), "should be person-based"); + Some((uid.id.to_string(), uid.expiry)) + } + + fn expires_at(read: Option<(String, ProviderExpiry)>) -> u64 { + match read.expect("should read the ID").1 { + ProviderExpiry::At(at) => at.get(), + other => panic!("should expire at a time, got {other:?}"), + } + } + + #[test] + fn reads_the_id_until_last_sync_plus_ttl() { + let cookie = json!({ + "connectId": "cid-example-1", + "he": "example-hashed-email", + "puid": "example-puid", + "lastSynced": 1_700_000_000_000_u64, + "lastUsed": 1_700_000_100_000_u64, + "ttl": 86_400_000, + }); + + let read = read(&cookie); + + assert_eq!( + read.as_ref().map(|(id, _)| id.as_str()), + Some("cid-example-1"), + "should store only connectId" + ); + assert_eq!( + expires_at(read), + 1_700_086_400, + "should expire at lastSynced + ttl" + ); + } + + #[test] + fn caps_the_ttl_and_defaults_a_missing_one() { + let long = json!({ "connectId": "cid", "lastSynced": 0, "ttl": 1e15 }); + let missing = json!({ "connectId": "cid", "lastSynced": 0 }); + + assert_eq!( + expires_at(read(&long)), + MAX_TTL.as_secs(), + "should cap at 14 days" + ); + assert_eq!( + expires_at(read(&missing)), + DEFAULT_TTL.as_secs(), + "should use the default" + ); + } + + #[test] + fn ignores_missing_malformed_and_opt_out_cookies() { + for cookie in [ + json!({}), + json!({ "connectId": "cid" }), + json!({ "connectId": "", "lastSynced": 1 }), + json!({ "connectId": "0", "lastSynced": 1 }), + json!({ "connectId": 5, "lastSynced": 1 }), + json!("not an object"), + ] { + assert!(read(&cookie).is_none(), "should ignore {cookie}"); + } + assert!( + provider() + .read_request_cookies(RequestCookies::new("connectId={broken")) + .is_none(), + "should ignore malformed JSON" + ); + } +} diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index 70eed9fe6..c1c59626b 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -10,6 +10,7 @@ //! depends on it. mod common; +pub mod connectid; pub mod id5; pub mod liveintent; pub mod liveramp; @@ -34,7 +35,10 @@ use trusted_server_core::settings::{ pub fn is_registered_kind(kind: &str) -> bool { matches!( kind, - id5::Id5::KIND | liveintent::LiveIntent::KIND | liveramp::LiveRamp::KIND + id5::Id5::KIND + | liveintent::LiveIntent::KIND + | liveramp::LiveRamp::KIND + | connectid::ConnectId::KIND ) } @@ -63,6 +67,7 @@ pub fn build( id5::Id5::KIND => build_kind::(config, secrets), liveintent::LiveIntent::KIND => build_kind::(config, secrets), liveramp::LiveRamp::KIND => build_kind::(config, secrets), + connectid::ConnectId::KIND => build_kind::(config, secrets), kind => Err(Report::new(ProviderConfigError::UnknownKind) .attach(format!("kind `{kind}` is not registered"))), } diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-connectid.md b/docs/superpowers/plans/2026-11-02-identity-provider-connectid.md new file mode 100644 index 000000000..07cff927c --- /dev/null +++ b/docs/superpowers/plans/2026-11-02-identity-provider-connectid.md @@ -0,0 +1,28 @@ +# Yahoo ConnectID Identity Provider Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Store Yahoo ConnectID from Yahoo's own first-party cookie, as an `[[ec.identity_providers]]` entry with `kind = "connectid"`. No login and no server call. + +**Architecture:** `crates/trusted-server-identity-providers/src/connectid.rs` holds `ConnectId` (a `ProviderKind`), `ConnectIdSettings` and `ConnectIdProvider`. The provider implements only `IdentityProvider::read_request_cookies`, using the cookie helpers in `common.rs`; `build_request` always fails, so no lookup is sent. + +**Spec:** `docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md`, section 7. + +## Cookie + +- Default cookie `connectId`; `cookie_name` overrides it. +- The value is JSON, optionally percent-encoded: `connectId`, `he`, `puid`, `lastSynced`, `lastUsed`, `ttl` (timestamps and TTL in milliseconds). +- Only `connectId`, `lastSynced` and `ttl` are read. `he` and `puid` are never deserialized, so they are never stored. +- A missing `connectId` or `lastSynced`, a non-string ID, a blank or opt-out ID, or malformed JSON reads as no ID. + +## Stored ID + +- One EID for the single configured source, one UID with `atype = 3`. +- Expiry `ProviderExpiry::At(lastSynced + min(ttl, 14 days))`; a missing `ttl` uses the shared default TTL. The provider reads no clock: the coordinator's validation drops the ID once that time has passed. + +## Tasks + +- [ ] Add `connectid.rs` and register the kind in `is_registered_kind` and `build`. +- [ ] Unit tests: ID and expiry from `lastSynced + ttl`, `he`/`puid` not stored, TTL capped and defaulted, missing, malformed and opt-out cookies ignored. +- [ ] Add a commented `kind = "connectid"` example to `trusted-server.example.toml`. +- [ ] Run `cargo test-fastly` and `cargo clippy-fastly`. diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 3f13c0649..afd21d3b8 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -193,6 +193,16 @@ pull_sync_concurrency = 3 # tcf_vendor_id = 9999 # [ec.identity_providers.settings] # cookie_names = ["_lr_env"] +# +# Yahoo ConnectID (kind "connectid"): no lookup and no endpoint. Reads +# ConnectID from Yahoo's own first-party `connectId` cookie and keeps it until +# its last sync plus TTL (at most 14 days). Takes exactly one source. +# [[ec.identity_providers]] +# id = "connectid" +# kind = "connectid" +# enabled = false +# sources = ["connectid.example.com"] +# tcf_vendor_id = 9999 # ============================================================================= From 5193e8aa54bb77ff0a32affaeefeb8fb4c9bf1a1 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 13:28:49 +0530 Subject: [PATCH 07/72] Add the UID2 identity provider Signed-off-by: dhruv8sh --- .../src/lib.rs | 3 + .../src/uid2.rs | 239 ++++++++++++++++++ .../2026-11-02-identity-provider-uid2.md | 33 +++ trusted-server.example.toml | 14 + 4 files changed, 289 insertions(+) create mode 100644 crates/trusted-server-identity-providers/src/uid2.rs create mode 100644 docs/superpowers/plans/2026-11-02-identity-provider-uid2.md diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index c1c59626b..d8eca7bab 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -14,6 +14,7 @@ pub mod connectid; pub mod id5; pub mod liveintent; pub mod liveramp; +pub mod uid2; use std::sync::Arc; use std::time::Duration; @@ -39,6 +40,7 @@ pub fn is_registered_kind(kind: &str) -> bool { | liveintent::LiveIntent::KIND | liveramp::LiveRamp::KIND | connectid::ConnectId::KIND + | uid2::Uid2::KIND ) } @@ -68,6 +70,7 @@ pub fn build( liveintent::LiveIntent::KIND => build_kind::(config, secrets), liveramp::LiveRamp::KIND => build_kind::(config, secrets), connectid::ConnectId::KIND => build_kind::(config, secrets), + uid2::Uid2::KIND => build_kind::(config, secrets), kind => Err(Report::new(ProviderConfigError::UnknownKind) .attach(format!("kind `{kind}` is not registered"))), } diff --git a/crates/trusted-server-identity-providers/src/uid2.rs b/crates/trusted-server-identity-providers/src/uid2.rs new file mode 100644 index 000000000..e5e95af02 --- /dev/null +++ b/crates/trusted-server-identity-providers/src/uid2.rs @@ -0,0 +1,239 @@ +//! UID2 identity provider: reads the UID2 advertising token from the +//! browser's own first-party cookies. +//! +//! The UID2 SDK stores its identity in the `__uid_2` cookie as +//! percent-encoded JSON. Only `advertising_token` and `identity_expires` +//! (milliseconds) are read: the token is stored with agent type 3 until +//! `identity_expires`, and the coordinator drops it once that time has +//! passed. The refresh token and response key are never read, so they are +//! never stored, and no refresh is made server side. +//! +//! When `__uid_2` yields nothing, Prebid's `__uid2_advertising_token` +//! cookie is read as a raw token with the shared default TTL. Lockr AIM can +//! write the token into a cookie whose name comes from its own +//! configuration; set `token_cookie_name` to it. +//! +//! Parsing is lenient: an undecodable cookie, a missing token or expiry, +//! or a blank or opt-out token reads as no ID. + +use std::borrow::Cow; + +use error_stack::Report; +use serde::Deserialize; +use trusted_server_core::ec::eid_types::{SourceDomain, UnixSeconds}; +use trusted_server_core::ec::provider::{ + IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, + ProviderConfigError, ProviderEid, ProviderError, ProviderExpiry, ProviderKind, ProviderRequest, + ProviderResponse, RequestCookies, +}; + +use crate::common::{ + bounded_expiry, cookie_capabilities, cookie_eid, is_usable_id, percent_decoded, +}; + +/// The UID2 provider kind. +pub struct Uid2; + +/// Settings for a `uid2` provider. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Uid2Settings { + /// Cookie holding the UID2 SDK identity JSON. + #[serde(default = "Uid2Settings::default_identity_cookie_name")] + pub identity_cookie_name: String, + /// Cookie holding a raw advertising token. + #[serde(default = "Uid2Settings::default_token_cookie_name")] + pub token_cookie_name: String, +} + +impl Uid2Settings { + fn default_identity_cookie_name() -> String { + "__uid_2".to_owned() + } + + fn default_token_cookie_name() -> String { + "__uid2_advertising_token".to_owned() + } +} + +/// A built UID2 provider. +#[derive(Debug)] +pub struct Uid2Provider { + capabilities: ProviderCapabilities, + source: SourceDomain, + identity_cookie_name: String, + token_cookie_name: String, +} + +impl ProviderKind for Uid2 { + const KIND: &'static str = "uid2"; + type Settings = Uid2Settings; + type Provider = Uid2Provider; + + fn build( + common: &ProviderCommon<'_>, + settings: Uid2Settings, + ) -> Result> { + let (capabilities, source) = cookie_capabilities(common, Self::KIND)?; + Ok(Uid2Provider { + capabilities, + source, + identity_cookie_name: settings.identity_cookie_name, + token_cookie_name: settings.token_cookie_name, + }) + } +} + +/// The fields of the `__uid_2` cookie this provider reads. +#[derive(Deserialize)] +struct IdentityCookie { + advertising_token: String, + identity_expires: u64, +} + +/// Decodes one `__uid_2` value into the token and its expiry. +fn identity(value: &str) -> Option<(String, ProviderExpiry)> { + let decoded = percent_decoded(value); + let cookie: IdentityCookie = serde_json::from_str(&decoded).ok()?; + if !is_usable_id(&cookie.advertising_token) { + return None; + } + let expires = UnixSeconds::new(cookie.identity_expires / 1000); + Some((cookie.advertising_token, ProviderExpiry::At(expires))) +} + +/// Decodes one raw advertising token cookie. +fn raw_token(value: &str) -> Option<(String, ProviderExpiry)> { + let token = percent_decoded(value); + let token = token.trim(); + is_usable_id(token).then(|| (token.to_owned(), bounded_expiry(None))) +} + +impl IdentityProvider for Uid2Provider { + fn capabilities(&self) -> &ProviderCapabilities { + &self.capabilities + } + + fn build_request( + &self, + _lookup: &LookupInput<'_>, + ) -> Result> { + Err(Report::new(ProviderError::Request).attach("uid2 makes no lookup")) + } + + fn parse_response<'body>( + &self, + _response: ProviderResponse<'body>, + ) -> Result, Report> { + Err(Report::new(ProviderError::InvalidResponse).attach("uid2 makes no lookup")) + } + + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> Option>> { + let (token, expiry) = cookies + .get(&self.identity_cookie_name) + .and_then(identity) + .or_else(|| cookies.get(&self.token_cookie_name).and_then(raw_token))?; + Some(vec![cookie_eid(&self.source, Cow::Owned(token), expiry)]) + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + use trusted_server_core::ec::provider::build_kind; + use trusted_server_core::settings::{IdentityProviderConfig, ProviderSecrets}; + + use super::*; + + fn provider(settings: &serde_json::Value) -> Box { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "uid2", + "kind": "uid2", + "sources": ["uid2.example.com"], + "settings": settings, + })) + .expect("should parse provider config"); + build_kind::(&config, &ProviderSecrets::default()).expect("should build provider") + } + + fn read(provider: &dyn IdentityProvider, header: &str) -> Option<(String, ProviderExpiry)> { + let eids = provider.read_request_cookies(RequestCookies::new(header))?; + assert_eq!(eids[0].source, "uid2.example.com", "should use the source"); + let uid = &eids[0].uids[0]; + assert_eq!(uid.atype, Some(3), "should be person-based"); + Some((uid.id.to_string(), uid.expiry)) + } + + fn identity_cookie(identity: &serde_json::Value) -> String { + format!("__uid_2={}", urlencoding::encode(&identity.to_string())) + } + + #[test] + fn reads_only_the_advertising_token_until_identity_expiry() { + let header = identity_cookie(&json!({ + "advertising_token": "AgAAAAexampletoken+/=", + "refresh_token": "example-refresh-token", + "identity_expires": 1_700_000_000_123_u64, + "refresh_from": 1_699_990_000_000_u64, + "refresh_expires": 1_702_000_000_000_u64, + "refresh_response_key": "example-response-key", + })); + + let (token, expiry) = read(&*provider(&json!({})), &header).expect("should read"); + + assert_eq!( + token, "AgAAAAexampletoken+/=", + "should store only the token" + ); + assert!( + matches!(expiry, ProviderExpiry::At(at) if at.get() == 1_700_000_000), + "should expire at identity_expires" + ); + } + + #[test] + fn falls_back_to_the_raw_token_cookie() { + let header = "__uid_2=%7Bbroken; __uid2_advertising_token=AgAAAAexampletoken"; + + let (token, expiry) = read(&*provider(&json!({})), header).expect("should read"); + + assert_eq!(token, "AgAAAAexampletoken", "should read the raw token"); + assert!( + matches!(expiry, ProviderExpiry::After(_)), + "should use the default TTL" + ); + } + + #[test] + fn reads_configured_cookie_names() { + let provider = provider(&json!({ "token_cookie_name": "example_lockr_uid2" })); + + let read = read(&*provider, "example_lockr_uid2=AgAAAAexampletoken"); + + assert_eq!( + read.map(|(token, _)| token).as_deref(), + Some("AgAAAAexampletoken"), + "should read the configured cookie" + ); + } + + #[test] + fn ignores_missing_malformed_and_opt_out_cookies() { + let provider = provider(&json!({})); + for header in [ + String::new(), + identity_cookie(&json!({ "advertising_token": "tok" })), + identity_cookie(&json!({ "identity_expires": 1 })), + identity_cookie(&json!({ "advertising_token": "", "identity_expires": 1 })), + "__uid2_advertising_token=0".to_owned(), + ] { + assert!( + read(&*provider, &header).is_none(), + "should ignore {header:?}" + ); + } + } +} diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md b/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md new file mode 100644 index 000000000..1b9d9922f --- /dev/null +++ b/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md @@ -0,0 +1,33 @@ +# UID2 Identity Provider Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Store the UID2 advertising token the browser already sends in first-party cookies, as an `[[ec.identity_providers]]` entry with `kind = "uid2"`. No login, no server call and no server-side refresh. + +**Architecture:** `crates/trusted-server-identity-providers/src/uid2.rs` holds `Uid2` (a `ProviderKind`), `Uid2Settings` and `Uid2Provider`. The provider implements only `IdentityProvider::read_request_cookies`, using the cookie helpers in `common.rs`; `build_request` always fails, so no lookup is sent. + +**Spec:** `docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md`, section 7. + +## Cookies + +- `identity_cookie_name` (default `__uid_2`): percent-encoded JSON from the UID2 SDK. Only `advertising_token` and `identity_expires` (milliseconds) are deserialized. `refresh_token`, `refresh_from`, `refresh_expires` and `refresh_response_key` are never read, so they are never stored. +- `token_cookie_name` (default `__uid2_advertising_token`): Prebid's raw token, read only when the identity cookie yields nothing. +- A missing token or expiry, a blank or opt-out token, or malformed JSON reads as no ID. + +## Stored ID + +- One EID for the single configured source, one UID with `atype = 3`. +- From the identity cookie: `ProviderExpiry::At(identity_expires)`; the coordinator's validation drops an expired token. +- From the raw token cookie: `bounded_expiry(None)`, the shared default TTL. +- No refresh: the browser SDK refreshes the identity and the next navigation with a due source picks up the new token. + +## Lockr + +Lockr AIM writes the UID2 token into a cookie named by its own configuration. Set `token_cookie_name` to that name. Lockr-delivered LiveRamp envelopes are handled by the `liveramp` kind's `cookie_names`, so no separate Lockr kind exists. Lockr-delivered ID5 cookies are not read; the `id5` kind resolves ID5 by lookup. + +## Tasks + +- [ ] Add `uid2.rs` and register the kind in `is_registered_kind` and `build`. +- [ ] Unit tests: token and expiry from `__uid_2` with refresh fields not stored, raw token fallback, configured cookie names, missing, malformed and opt-out cookies ignored. +- [ ] Add a commented `kind = "uid2"` example to `trusted-server.example.toml`. +- [ ] Run `cargo test-fastly` and `cargo clippy-fastly`. diff --git a/trusted-server.example.toml b/trusted-server.example.toml index afd21d3b8..3eec005d1 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -203,6 +203,20 @@ pull_sync_concurrency = 3 # enabled = false # sources = ["connectid.example.com"] # tcf_vendor_id = 9999 +# +# UID2 (kind "uid2"): no lookup and no endpoint. Reads the advertising token +# from the UID2 SDK's `__uid_2` cookie until its `identity_expires`, falling +# back to Prebid's raw `__uid2_advertising_token` cookie. Never stores or uses +# the refresh token. Takes exactly one source. Set `token_cookie_name` to the +# cookie Lockr AIM writes the token into, if you use Lockr. +# [[ec.identity_providers]] +# id = "uid2" +# kind = "uid2" +# enabled = false +# sources = ["uid2.example.com"] +# [ec.identity_providers.settings] +# identity_cookie_name = "__uid_2" +# token_cookie_name = "__uid2_advertising_token" # ============================================================================= From fc9fdd07cda823b58dc0f0bcad1a632a6ec4aabc Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 14:47:42 +0530 Subject: [PATCH 08/72] Read the ID5 ID from its first-party cookie Signed-off-by: dhruv8sh --- .../src/id5.rs | 322 +++++++++++++++--- .../plans/2026-11-02-identity-provider-id5.md | 12 +- trusted-server.example.toml | 4 + 3 files changed, 290 insertions(+), 48 deletions(-) diff --git a/crates/trusted-server-identity-providers/src/id5.rs b/crates/trusted-server-identity-providers/src/id5.rs index ab1f5add9..99a0e8a8b 100644 --- a/crates/trusted-server-identity-providers/src/id5.rs +++ b/crates/trusted-server-identity-providers/src/id5.rs @@ -11,6 +11,16 @@ //! `cache_control.max_age_sec` is bounded by [`bounded_expiry`]: a missing //! value gets the default TTL and a zero or tiny one the minimum TTL. //! +//! Before any lookup, the provider reads ID5's first-party cookies listed +//! in `cookie_names` (default `id5id`), in order. Prebid's ID5 module and +//! Lockr AIM write them: a cookie holding the ID5 fetch response JSON +//! (plain or percent-encoded) is parsed exactly like a lookup response, and +//! a bare ID string becomes one UID on the first source with agent type 1 +//! and the default TTL. The response `signature` is never stored. When a +//! percent-encoded JSON cookie is read, its `ext` objects are dropped, +//! since they cannot outlive the decoded copy. A cookie that yields no +//! usable UID is ignored and the lookup runs as usual. +//! //! The request field names for the client IP and user agent follow ID5's //! server-to-server integration and should be confirmed with ID5 for each //! partner account. @@ -29,13 +39,15 @@ use trusted_server_core::ec::eid_types::SourceDomain; use trusted_server_core::ec::provider::{ EidLevelFields, EidShape, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, ProviderConfigError, ProviderEid, ProviderError, ProviderKind, ProviderRequest, - ProviderResponse, ProviderUid, SourceCapability, SourceInput, + ProviderResponse, ProviderUid, RequestCookies, SourceCapability, SourceInput, }; use url::Url; use serde_json::value::RawValue; -use crate::common::{VendorEid, bounded_expiry, is_routable_ip, join_section_ids}; +use crate::common::{ + VendorEid, bounded_expiry, is_routable_ip, is_usable_id, join_section_ids, percent_decoded, +}; /// The UID ID5 returns for a visitor who opted out. const OPTED_OUT_UID: &str = "0"; @@ -55,6 +67,16 @@ pub struct Id5; pub struct Id5Settings { /// The ID5 partner number assigned to the publisher. pub partner_id: u32, + /// First-party cookies read in order before any lookup; the first + /// yielding a usable UID wins. + #[serde(default = "Id5Settings::default_cookie_names")] + pub cookie_names: Vec, +} + +impl Id5Settings { + fn default_cookie_names() -> Vec { + vec!["id5id".to_owned()] + } } /// A built ID5 provider. @@ -63,6 +85,7 @@ pub struct Id5Provider { capabilities: ProviderCapabilities, url: Url, partner_id: u32, + cookie_names: Box<[String]>, } impl ProviderKind for Id5 { @@ -108,6 +131,7 @@ impl ProviderKind for Id5 { }, url, partner_id: settings.partner_id, + cookie_names: settings.cookie_names.into_boxed_slice(), }) } } @@ -173,6 +197,93 @@ impl Id5Provider { .map(|known| &known.source) .find(|known| **known == source) } + + /// Parses an ID5 fetch reply into its kept EIDs. + fn parse_reply<'body>( + &self, + body: &'body [u8], + ) -> Result>, serde_json::Error> { + let reply: FetchReply<'body> = serde_json::from_slice(body)?; + let max_age = reply + .cache_control + .and_then(|raw| serde_json::from_str::(raw.get()).ok()) + .and_then(|cache| cache.max_age_sec) + .map(Duration::from_secs); + let expiry = bounded_expiry(max_age); + + let entries: BTreeMap = reply + .ids + .and_then(|raw| serde_json::from_str(raw.get()).ok()) + .unwrap_or_default(); + let mut eids: Vec> = entries + .into_values() + .filter_map(|raw| serde_json::from_str::>(raw.get()).ok()) + .filter_map(|id| id.eid) + .filter_map(|mut eid| { + let source = self.known_source(&eid.source)?; + eid.source = Cow::Owned(source.as_str().to_owned()); + eid.uids.retain(|uid| is_usable_uid(&uid.id)); + (!eid.uids.is_empty()).then(|| eid.into_provider_eid(expiry)) + }) + .collect(); + let primary = self.primary_source().as_str(); + let has_primary = eids.iter().any(|eid| eid.source == primary); + let universal_uid = reply + .universal_uid + .and_then(|raw| serde_json::from_str::>(raw.get()).ok()) + .filter(|uid| is_usable_uid(uid)); + if !has_primary && let Some(uid) = universal_uid { + eids.push(ProviderEid { + source: Cow::Owned(primary.to_owned()), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids: vec![ProviderUid { + id: uid, + atype: Some(1), + ext: None, + expiry, + }], + }); + } + Ok(eids) + } + + /// Reads one ID5 cookie value: a fetch reply JSON or a bare ID. + fn cookie_eids<'cookie>(&self, value: &'cookie str) -> Option>> { + let eids = match percent_decoded(value) { + Cow::Borrowed(value) if value.trim_start().starts_with('{') => { + self.parse_reply(value.as_bytes()).ok()? + } + Cow::Owned(value) if value.trim_start().starts_with('{') => self + .parse_reply(value.as_bytes()) + .ok()? + .into_iter() + .map(owned_without_ext) + .collect(), + value => { + let id = value.trim(); + if !is_usable_uid(id) || !is_usable_id(id) { + return None; + } + vec![ProviderEid { + source: Cow::Owned(self.primary_source().as_str().to_owned()), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids: vec![ProviderUid { + id: Cow::Owned(id.to_owned()), + atype: Some(1), + ext: None, + expiry: bounded_expiry(None), + }], + }] + } + }; + (!eids.is_empty()).then_some(eids) + } } impl IdentityProvider for Id5Provider { @@ -227,57 +338,47 @@ impl IdentityProvider for Id5Provider { return Err(Report::new(ProviderError::InvalidResponse) .attach(format!("ID5 returned status {}", response.status))); } - let reply: FetchReply<'body> = - serde_json::from_slice(response.body).change_context(ProviderError::InvalidResponse)?; - let max_age = reply - .cache_control - .and_then(|raw| serde_json::from_str::(raw.get()).ok()) - .and_then(|cache| cache.max_age_sec) - .map(Duration::from_secs); - let expiry = bounded_expiry(max_age); - - let entries: BTreeMap = reply - .ids - .and_then(|raw| serde_json::from_str(raw.get()).ok()) - .unwrap_or_default(); - let mut eids: Vec> = entries - .into_values() - .filter_map(|raw| serde_json::from_str::>(raw.get()).ok()) - .filter_map(|id| id.eid) - .filter_map(|mut eid| { - let source = self.known_source(&eid.source)?; - eid.source = Cow::Owned(source.as_str().to_owned()); - eid.uids.retain(|uid| is_usable_uid(&uid.id)); - (!eid.uids.is_empty()).then(|| eid.into_provider_eid(expiry)) - }) - .collect(); - let primary = self.primary_source().as_str(); - let has_primary = eids.iter().any(|eid| eid.source == primary); - let universal_uid = reply - .universal_uid - .and_then(|raw| serde_json::from_str::>(raw.get()).ok()) - .filter(|uid| is_usable_uid(uid)); - if !has_primary && let Some(uid) = universal_uid { - eids.push(ProviderEid { - source: Cow::Owned(primary.to_owned()), - inserter: None, - matcher: None, - mm: None, - ext: None, - uids: vec![ProviderUid { - id: uid, - atype: Some(1), - ext: None, - expiry, - }], - }); - } + let eids = self + .parse_reply(response.body) + .change_context(ProviderError::InvalidResponse)?; Ok(if eids.is_empty() { LookupOutcome::NoMatch } else { LookupOutcome::Found(eids) }) } + + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> Option>> { + self.cookie_names + .iter() + .filter_map(|name| cookies.get(name)) + .find_map(|value| self.cookie_eids(value)) + } +} + +/// Copies an EID parsed from a temporary string, dropping its `ext` +/// objects, which cannot be copied out of the borrowed JSON. +fn owned_without_ext<'any>(eid: ProviderEid<'_>) -> ProviderEid<'any> { + ProviderEid { + source: Cow::Owned(eid.source.into_owned()), + inserter: eid.inserter.map(|value| Cow::Owned(value.into_owned())), + matcher: eid.matcher.map(|value| Cow::Owned(value.into_owned())), + mm: eid.mm, + ext: None, + uids: eid + .uids + .into_iter() + .map(|uid| ProviderUid { + id: Cow::Owned(uid.id.into_owned()), + atype: uid.atype, + ext: None, + expiry: uid.expiry, + }) + .collect(), + } } /// Returns whether `id` is a real UID: not empty and not the opt-out UID. @@ -523,4 +624,131 @@ mod tests { ); } } + + /// Source, UID, agent type, whether a UID `ext` is kept, and TTL. + type ReadUid = (String, String, Option, bool, Duration); + + fn read(provider: &dyn IdentityProvider, header: &str) -> Option> { + provider + .read_request_cookies(RequestCookies::new(header)) + .map(|eids| { + eids.into_iter() + .flat_map(|eid| { + let source = eid.source.into_owned(); + eid.uids.into_iter().map(move |uid| { + let ProviderExpiry::After(ttl) = uid.expiry else { + panic!("should state a relative expiry"); + }; + ( + source.clone(), + uid.id.into_owned(), + uid.atype, + uid.ext.is_some(), + ttl, + ) + }) + }) + .collect() + }) + } + + fn reply_cookie() -> serde_json::Value { + json!({ + "universal_uid": "legacy-example", + "signature": "signature-example", + "ids": { + "id5": { "eid": { "source": "id5.example.com", "uids": [{ "id": "id5-example", "atype": 1, "ext": { "linkType": 2 } }] } }, + }, + "cache_control": { "max_age_sec": 7200 }, + }) + } + + #[test] + fn reads_a_plain_json_reply_cookie() { + let provider = provider(); + let header = format!("other=1; id5id={}", reply_cookie()); + + let uids = read(&*provider, &header).expect("should read the cookie"); + + assert_eq!( + uids, + vec![( + "id5.example.com".to_owned(), + "id5-example".to_owned(), + Some(1), + true, + Duration::from_secs(7200) + )], + "should parse the reply like a lookup response and never store the signature" + ); + } + + #[test] + fn reads_a_percent_encoded_json_reply_cookie() { + let provider = provider(); + let header = format!("id5id={}", urlencoding::encode(&reply_cookie().to_string())); + + let uids = read(&*provider, &header).expect("should read the cookie"); + + assert_eq!( + uids, + vec![( + "id5.example.com".to_owned(), + "id5-example".to_owned(), + Some(1), + false, + Duration::from_secs(7200) + )], + "should decode the reply and drop ext objects" + ); + } + + #[test] + fn reads_a_bare_id_cookie_from_a_configured_name() { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "id5", + "kind": "id5", + "endpoint": "https://id5.example.com/g/v2", + "allowed_domains": ["id5.example.com"], + "sources": ["id5.example.com"], + "settings": { "partner_id": 1234, "cookie_names": ["example_lockr_id5", "id5id"] }, + })) + .expect("should parse provider config"); + let provider = + build_kind::(&config, &ProviderSecrets::default()).expect("should build provider"); + + let uids = read(&provider, "id5id=0; example_lockr_id5=bare-example") + .expect("should read the bare ID"); + + assert_eq!( + uids, + vec![( + "id5.example.com".to_owned(), + "bare-example".to_owned(), + Some(1), + false, + crate::common::DEFAULT_TTL + )], + "should store the bare ID on the first source with the default TTL" + ); + } + + #[test] + fn ignores_opted_out_and_malformed_cookies_so_the_lookup_runs() { + let provider = provider(); + let opted_out = json!({ "universal_uid": "0", "ids": {} }).to_string(); + + for header in [ + "id5id=0".to_owned(), + format!("id5id={opted_out}"), + format!("id5id={}", urlencoding::encode(&opted_out)), + "id5id={not json".to_owned(), + "other=bare-example".to_owned(), + ] { + assert!( + read(&*provider, &header).is_none(), + "should read no ID from {header}" + ); + } + } } diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-id5.md b/docs/superpowers/plans/2026-11-02-identity-provider-id5.md index 68fcc116e..81eb5c323 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-id5.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-id5.md @@ -29,10 +29,20 @@ - `cache_control.max_age_sec` becomes `ProviderExpiry::After` for every UID. - With no EID for the first source, a non-empty, non-`0` `universal_uid` becomes an EID with `atype = 1`. +## First-party cookie + +ID5's source is provider-owned, so the browser `ts-eids` copy is dropped. Prebid's ID5 module and Lockr AIM store the ID in a first-party cookie instead, which `read_request_cookies` reads before any lookup. + +- Setting `cookie_names` (default `["id5id"]`), tried in order; Lockr's key name is configurable, so operators list it. +- A value that is JSON after percent-decoding is parsed with the same code as a lookup response (lenient entries, opt-out `0` dropped, normalized sources, `cache_control.max_age_sec` bounded). `ext` objects are dropped when the value was percent-encoded, since they borrow from the decoded copy. +- Any other value is a bare ID: one UID on the first source with `atype = 1` and the default TTL. +- `signature` is never stored as a UID. A cookie that yields no usable UID is ignored, and the lookup runs as usual; a kept UID skips the lookup. + ## Tasks - [ ] Add `common.rs` with `is_routable_ip`, `join_section_ids` and the borrowed `VendorEid`. - [ ] Add `id5.rs` and the `build` arm. - [ ] Unit tests: request shape, private IP dropped, complete EIDs parsed and unconfigured sources dropped, legacy fallback, opted-out and empty replies are no match, error status and malformed body rejected. -- [ ] Add a commented `kind = "id5"` example to `trusted-server.example.toml`. +- [ ] Read `cookie_names` in `read_request_cookies`; tests for plain and percent-encoded reply JSON, a bare ID from a configured name, and opted-out or malformed cookies reading as no ID. +- [ ] Add a commented `kind = "id5"` example, with `cookie_names`, to `trusted-server.example.toml`. - [ ] Run `cargo test-fastly` and `cargo clippy-fastly`. diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 3eec005d1..4aad6e7c3 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -153,6 +153,9 @@ pull_sync_concurrency = 3 # # ID5 (kind "id5"): anonymous lookup, no secrets. The first source receives # the legacy `universal_uid`. Use the endpoint and sources ID5 gives you. +# Before any lookup it reads the first-party cookies in `cookie_names`, in +# order (Prebid's ID5 module writes `id5id`; Lockr AIM writes the ID under a +# key name from its own configuration, so list that name too). # [[ec.identity_providers]] # id = "id5" # kind = "id5" @@ -163,6 +166,7 @@ pull_sync_concurrency = 3 # tcf_vendor_id = 9999 # [ec.identity_providers.settings] # partner_id = 1234 +# cookie_names = ["id5id"] # # LiveIntent (kind "liveintent"): anonymous lookup, no secrets. Each attribute # LiveIntent returns is stored under the source it maps to; every mapped From fceef645ef6d26132b70e0f98919b37a40a3c0e5 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 14:50:27 +0530 Subject: [PATCH 09/72] Pass the configured provider ID to identity provider builds Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/provider.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs index eb3373071..151ebc853 100644 --- a/crates/trusted-server-core/src/ec/provider.rs +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -451,6 +451,8 @@ pub trait ProviderKind { /// Shared configuration every provider receives at build time. #[derive(Debug, Clone, Copy)] pub struct ProviderCommon<'a> { + /// Configured provider ID, for logs that must tell instances apart. + pub id: &'a str, /// Configured lookup endpoint, absent for cookie-only kinds. pub endpoint: Option<&'a Url>, /// Configured sources. @@ -489,6 +491,7 @@ pub fn build_kind( let endpoint = parse_endpoint(config)?; let sources = parse_sources(config)?; let common = ProviderCommon { + id: &config.id, endpoint: endpoint.as_ref(), sources: &sources, secrets, From f5a75fc5f2c7af573b1c7cbd0fb8384b6db29049 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 14:51:18 +0530 Subject: [PATCH 10/72] Use cookie name lists across cookie-reading providers Signed-off-by: dhruv8sh --- .../src/connectid.rs | 25 +++++++++++------ .../src/uid2.rs | 28 +++++++++++-------- .../2026-11-02-identity-provider-connectid.md | 2 +- .../2026-11-02-identity-provider-uid2.md | 4 +-- trusted-server.example.toml | 8 ++++-- 5 files changed, 42 insertions(+), 25 deletions(-) diff --git a/crates/trusted-server-identity-providers/src/connectid.rs b/crates/trusted-server-identity-providers/src/connectid.rs index aaddfeffd..16b78ef36 100644 --- a/crates/trusted-server-identity-providers/src/connectid.rs +++ b/crates/trusted-server-identity-providers/src/connectid.rs @@ -36,14 +36,15 @@ pub struct ConnectId; #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] pub struct ConnectIdSettings { - /// Cookie holding the `ConnectID` object. - #[serde(default = "ConnectIdSettings::default_cookie_name")] - pub cookie_name: String, + /// Cookies holding the `ConnectID` object, read in order; the first + /// holding a usable ID wins. + #[serde(default = "ConnectIdSettings::default_cookie_names")] + pub cookie_names: Vec, } impl ConnectIdSettings { - fn default_cookie_name() -> String { - "connectId".to_owned() + fn default_cookie_names() -> Vec { + vec!["connectId".to_owned()] } } @@ -52,7 +53,7 @@ impl ConnectIdSettings { pub struct ConnectIdProvider { capabilities: ProviderCapabilities, source: SourceDomain, - cookie_name: String, + cookie_names: Box<[String]>, } impl ProviderKind for ConnectId { @@ -64,11 +65,15 @@ impl ProviderKind for ConnectId { common: &ProviderCommon<'_>, settings: ConnectIdSettings, ) -> Result> { + if settings.cookie_names.is_empty() { + return Err(Report::new(ProviderConfigError::Invalid) + .attach("a connectid provider needs at least one cookie name")); + } let (capabilities, source) = cookie_capabilities(common, Self::KIND)?; Ok(ConnectIdProvider { capabilities, source, - cookie_name: settings.cookie_name, + cookie_names: settings.cookie_names.into_boxed_slice(), }) } } @@ -127,7 +132,11 @@ impl IdentityProvider for ConnectIdProvider { &self, cookies: RequestCookies<'cookie>, ) -> Option>> { - let (id, expiry) = connect_id(cookies.get(&self.cookie_name)?)?; + let (id, expiry) = self + .cookie_names + .iter() + .filter_map(|name| cookies.get(name)) + .find_map(connect_id)?; Some(vec![cookie_eid(&self.source, Cow::Owned(id), expiry)]) } } diff --git a/crates/trusted-server-identity-providers/src/uid2.rs b/crates/trusted-server-identity-providers/src/uid2.rs index e5e95af02..d010337dd 100644 --- a/crates/trusted-server-identity-providers/src/uid2.rs +++ b/crates/trusted-server-identity-providers/src/uid2.rs @@ -9,9 +9,9 @@ //! never stored, and no refresh is made server side. //! //! When `__uid_2` yields nothing, Prebid's `__uid2_advertising_token` -//! cookie is read as a raw token with the shared default TTL. Lockr AIM can +//! cookie (or each name in `cookie_names`) is read as a raw token with the shared default TTL. Lockr AIM can //! write the token into a cookie whose name comes from its own -//! configuration; set `token_cookie_name` to it. +//! configuration; list that name in `cookie_names`. //! //! Parsing is lenient: an undecodable cookie, a missing token or expiry, //! or a blank or opt-out token reads as no ID. @@ -41,9 +41,10 @@ pub struct Uid2Settings { /// Cookie holding the UID2 SDK identity JSON. #[serde(default = "Uid2Settings::default_identity_cookie_name")] pub identity_cookie_name: String, - /// Cookie holding a raw advertising token. - #[serde(default = "Uid2Settings::default_token_cookie_name")] - pub token_cookie_name: String, + /// Cookies holding a raw advertising token, read in order after the + /// identity cookie. + #[serde(default = "Uid2Settings::default_cookie_names")] + pub cookie_names: Vec, } impl Uid2Settings { @@ -51,8 +52,8 @@ impl Uid2Settings { "__uid_2".to_owned() } - fn default_token_cookie_name() -> String { - "__uid2_advertising_token".to_owned() + fn default_cookie_names() -> Vec { + vec!["__uid2_advertising_token".to_owned()] } } @@ -62,7 +63,7 @@ pub struct Uid2Provider { capabilities: ProviderCapabilities, source: SourceDomain, identity_cookie_name: String, - token_cookie_name: String, + cookie_names: Box<[String]>, } impl ProviderKind for Uid2 { @@ -79,7 +80,7 @@ impl ProviderKind for Uid2 { capabilities, source, identity_cookie_name: settings.identity_cookie_name, - token_cookie_name: settings.token_cookie_name, + cookie_names: settings.cookie_names.into_boxed_slice(), }) } } @@ -135,7 +136,12 @@ impl IdentityProvider for Uid2Provider { let (token, expiry) = cookies .get(&self.identity_cookie_name) .and_then(identity) - .or_else(|| cookies.get(&self.token_cookie_name).and_then(raw_token))?; + .or_else(|| { + self.cookie_names + .iter() + .filter_map(|name| cookies.get(name)) + .find_map(raw_token) + })?; Some(vec![cookie_eid(&self.source, Cow::Owned(token), expiry)]) } } @@ -209,7 +215,7 @@ mod tests { #[test] fn reads_configured_cookie_names() { - let provider = provider(&json!({ "token_cookie_name": "example_lockr_uid2" })); + let provider = provider(&json!({ "cookie_names": ["example_lockr_uid2"] })); let read = read(&*provider, "example_lockr_uid2=AgAAAAexampletoken"); diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-connectid.md b/docs/superpowers/plans/2026-11-02-identity-provider-connectid.md index 07cff927c..05404d501 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-connectid.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-connectid.md @@ -10,7 +10,7 @@ ## Cookie -- Default cookie `connectId`; `cookie_name` overrides it. +- Default cookie `connectId`; `cookie_names` overrides the list, read in order. - The value is JSON, optionally percent-encoded: `connectId`, `he`, `puid`, `lastSynced`, `lastUsed`, `ttl` (timestamps and TTL in milliseconds). - Only `connectId`, `lastSynced` and `ttl` are read. `he` and `puid` are never deserialized, so they are never stored. - A missing `connectId` or `lastSynced`, a non-string ID, a blank or opt-out ID, or malformed JSON reads as no ID. diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md b/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md index 1b9d9922f..6a6389571 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md @@ -11,7 +11,7 @@ ## Cookies - `identity_cookie_name` (default `__uid_2`): percent-encoded JSON from the UID2 SDK. Only `advertising_token` and `identity_expires` (milliseconds) are deserialized. `refresh_token`, `refresh_from`, `refresh_expires` and `refresh_response_key` are never read, so they are never stored. -- `token_cookie_name` (default `__uid2_advertising_token`): Prebid's raw token, read only when the identity cookie yields nothing. +- `cookie_names` (default `["__uid2_advertising_token"]`): cookies holding a raw token, such as Prebid's, read in order only when the identity cookie yields nothing. - A missing token or expiry, a blank or opt-out token, or malformed JSON reads as no ID. ## Stored ID @@ -23,7 +23,7 @@ ## Lockr -Lockr AIM writes the UID2 token into a cookie named by its own configuration. Set `token_cookie_name` to that name. Lockr-delivered LiveRamp envelopes are handled by the `liveramp` kind's `cookie_names`, so no separate Lockr kind exists. Lockr-delivered ID5 cookies are not read; the `id5` kind resolves ID5 by lookup. +Lockr AIM writes the UID2 token into a cookie named by its own configuration. List that name in `cookie_names`. Lockr-delivered LiveRamp envelopes are handled by the `liveramp` kind's `cookie_names`, so no separate Lockr kind exists. Lockr-delivered ID5 cookies are not read; the `id5` kind resolves ID5 by lookup. ## Tasks diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 4aad6e7c3..573b07890 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -207,12 +207,14 @@ pull_sync_concurrency = 3 # enabled = false # sources = ["connectid.example.com"] # tcf_vendor_id = 9999 +# [ec.identity_providers.settings] +# cookie_names = ["connectId"] # # UID2 (kind "uid2"): no lookup and no endpoint. Reads the advertising token # from the UID2 SDK's `__uid_2` cookie until its `identity_expires`, falling # back to Prebid's raw `__uid2_advertising_token` cookie. Never stores or uses -# the refresh token. Takes exactly one source. Set `token_cookie_name` to the -# cookie Lockr AIM writes the token into, if you use Lockr. +# the refresh token. Takes exactly one source. `cookie_names` lists the raw +# token cookies, read in order after `identity_cookie_name`. # [[ec.identity_providers]] # id = "uid2" # kind = "uid2" @@ -220,7 +222,7 @@ pull_sync_concurrency = 3 # sources = ["uid2.example.com"] # [ec.identity_providers.settings] # identity_cookie_name = "__uid_2" -# token_cookie_name = "__uid2_advertising_token" +# cookie_names = ["__uid2_advertising_token"] # ============================================================================= From 8ce11105c4b311bdfcffd34a0a1499f676ca22ce Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 14:53:32 +0530 Subject: [PATCH 11/72] Add a Lockr cookie capability for identity providers Signed-off-by: dhruv8sh --- .../src/common.rs | 117 +++++++++++++++++- .../src/connectid.rs | 17 +-- .../src/id5.rs | 65 ++++++++-- .../src/lib.rs | 83 +++++++++++++ .../src/liveintent.rs | 3 + .../src/liveramp.rs | 70 +++++++++-- .../src/uid2.rs | 65 ++++++++-- .../2026-11-02-identity-provider-connectid.md | 2 + .../plans/2026-11-02-identity-provider-id5.md | 5 +- ...2026-11-02-identity-provider-liveintent.md | 2 + .../2026-11-02-identity-provider-liveramp.md | 2 +- ...1-02-identity-provider-lockr-capability.md | 24 ++++ .../2026-11-02-identity-provider-uid2.md | 2 +- trusted-server.example.toml | 26 +++- 14 files changed, 432 insertions(+), 51 deletions(-) create mode 100644 docs/superpowers/plans/2026-11-02-identity-provider-lockr-capability.md diff --git a/crates/trusted-server-identity-providers/src/common.rs b/crates/trusted-server-identity-providers/src/common.rs index 242a0ba17..2dc5138de 100644 --- a/crates/trusted-server-identity-providers/src/common.rs +++ b/crates/trusted-server-identity-providers/src/common.rs @@ -11,7 +11,8 @@ use serde_json::value::RawValue; use trusted_server_core::ec::eid_types::SourceDomain; use trusted_server_core::ec::provider::{ EidLevelFields, EidShape, ProviderCapabilities, ProviderCommon, ProviderConfigError, - ProviderEid, ProviderExpiry, ProviderUid, SourceCapability, SourceInput, + ProviderEid, ProviderExpiry, ProviderKind, ProviderUid, RequestCookies, SourceCapability, + SourceInput, }; /// Returns whether `ip` is a public address worth sending to a vendor. @@ -185,6 +186,120 @@ pub(crate) fn percent_decoded(value: &str) -> Cow<'_, str> { urlencoding::decode(value).unwrap_or(Cow::Borrowed(value)) } +/// Settings for the Lockr capability, under a provider's +/// `[ec.identity_providers.settings.lockr]`. +/// +/// Lockr AIM writes other vendors' IDs into first-party cookies on the +/// publisher domain, under key names configured per publisher in Lockr. +/// Only kinds implementing [`ReadsLockr`] accept this table; the settings +/// of every other kind reject it as an unknown field. +#[derive(Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LockrSettings { + /// Whether the Lockr cookies are read. Off by default. + #[serde(default)] + pub enabled: bool, + /// Lockr cookies read in order, after the vendor's own cookies. + /// Defaults to the vendor's known Lockr key. + #[serde(default)] + pub cookie_names: Option>, +} + +/// The Lockr capability: declared by provider kinds whose IDs Lockr AIM +/// delivers. +pub(crate) trait ReadsLockr: ProviderKind { + /// The key names Lockr uses for this vendor by default. + const LOCKR_COOKIE_NAMES: &'static [&'static str]; +} + +/// Reads a provider's cookie-carried ID: the vendor's own cookies in order, +/// then, when the Lockr capability is enabled, the Lockr cookies in order. +/// +/// Each value goes to the provider's own parser, so Lockr reuses it. +#[derive(Debug)] +pub(crate) struct CookieReader { + provider_id: String, + own: Box<[String]>, + lockr: Box<[String]>, +} + +impl CookieReader { + /// A reader for a kind without the Lockr capability. + pub(crate) fn own(common: &ProviderCommon<'_>, own: Vec) -> Self { + Self { + provider_id: common.id.to_owned(), + own: own.into_boxed_slice(), + lockr: Box::default(), + } + } + + /// A reader for a kind with the Lockr capability. Lockr names already + /// in `own` are skipped, since they were read first. + /// + /// # Errors + /// + /// Returns [`ProviderConfigError::Invalid`] when Lockr is enabled with + /// an empty `cookie_names` list. + pub(crate) fn with_lockr( + common: &ProviderCommon<'_>, + own: Vec, + lockr: LockrSettings, + ) -> Result> { + let mut reader = Self::own(common, own); + if !lockr.enabled { + return Ok(reader); + } + let names = lockr.cookie_names.unwrap_or_else(|| { + K::LOCKR_COOKIE_NAMES + .iter() + .map(|name| (*name).to_owned()) + .collect() + }); + if names.is_empty() { + return Err(Report::new(ProviderConfigError::Invalid).attach(format!( + "lockr for a {} provider needs at least one cookie name", + K::KIND + ))); + } + reader.lockr = names + .into_iter() + .filter(|name| !reader.own.contains(name)) + .collect(); + Ok(reader) + } + + /// Whether any cookie name is configured. + pub(crate) fn is_empty(&self) -> bool { + self.own.is_empty() && self.lockr.is_empty() + } + + /// Returns the first value `parse` accepts, trying the own cookies + /// before the Lockr cookies. + pub(crate) fn read<'cookie, T>( + &self, + cookies: RequestCookies<'cookie>, + mut parse: impl FnMut(&'cookie str) -> Option, + ) -> Option { + if let Some(found) = self + .own + .iter() + .filter_map(|name| cookies.get(name)) + .find_map(&mut parse) + { + return Some(found); + } + let (name, found) = self.lockr.iter().find_map(|name| { + let found = parse(cookies.get(name)?)?; + Some((name, found)) + })?; + log::debug!( + "Identity provider '{}' reads its ID from cookie '{name}' (origin lockr)", + self.provider_id + ); + Some(found) + } +} + /// A complete `OpenRTB` EID as a vendor returns it, borrowed from the body. /// /// Unknown fields are ignored so one new vendor field does not fail the diff --git a/crates/trusted-server-identity-providers/src/connectid.rs b/crates/trusted-server-identity-providers/src/connectid.rs index 16b78ef36..8013639f3 100644 --- a/crates/trusted-server-identity-providers/src/connectid.rs +++ b/crates/trusted-server-identity-providers/src/connectid.rs @@ -8,6 +8,9 @@ //! drops it once that time has passed. A missing `ttl` uses the shared //! default TTL. No server call is made. //! +//! This kind has no Lockr capability: Lockr AIM is not documented to +//! deliver `ConnectID`, so a `lockr` settings table is rejected. +//! //! The cookie's `he` and `puid` fields are never read, so they are never //! stored. Parsing is lenient: an undecodable cookie, a missing //! `connectId` or `lastSynced`, or a blank or opt-out ID reads as no ID. @@ -24,7 +27,9 @@ use trusted_server_core::ec::provider::{ ProviderResponse, RequestCookies, }; -use crate::common::{DEFAULT_TTL, cookie_capabilities, cookie_eid, is_usable_id, percent_decoded}; +use crate::common::{ + CookieReader, DEFAULT_TTL, cookie_capabilities, cookie_eid, is_usable_id, percent_decoded, +}; /// Longest time a `ConnectID` is kept after its last sync. pub const MAX_TTL: Duration = Duration::from_secs(14 * 24 * 60 * 60); @@ -53,7 +58,7 @@ impl ConnectIdSettings { pub struct ConnectIdProvider { capabilities: ProviderCapabilities, source: SourceDomain, - cookie_names: Box<[String]>, + cookies: CookieReader, } impl ProviderKind for ConnectId { @@ -73,7 +78,7 @@ impl ProviderKind for ConnectId { Ok(ConnectIdProvider { capabilities, source, - cookie_names: settings.cookie_names.into_boxed_slice(), + cookies: CookieReader::own(common, settings.cookie_names), }) } } @@ -132,11 +137,7 @@ impl IdentityProvider for ConnectIdProvider { &self, cookies: RequestCookies<'cookie>, ) -> Option>> { - let (id, expiry) = self - .cookie_names - .iter() - .filter_map(|name| cookies.get(name)) - .find_map(connect_id)?; + let (id, expiry) = self.cookies.read(cookies, connect_id)?; Some(vec![cookie_eid(&self.source, Cow::Owned(id), expiry)]) } } diff --git a/crates/trusted-server-identity-providers/src/id5.rs b/crates/trusted-server-identity-providers/src/id5.rs index 99a0e8a8b..15b1ca86a 100644 --- a/crates/trusted-server-identity-providers/src/id5.rs +++ b/crates/trusted-server-identity-providers/src/id5.rs @@ -12,8 +12,9 @@ //! value gets the default TTL and a zero or tiny one the minimum TTL. //! //! Before any lookup, the provider reads ID5's first-party cookies listed -//! in `cookie_names` (default `id5id`), in order. Prebid's ID5 module and -//! Lockr AIM write them: a cookie holding the ID5 fetch response JSON +//! in `cookie_names` (default `id5id`, written by Prebid's ID5 module), in +//! order, then, when the Lockr capability is enabled, the cookies Lockr AIM +//! writes (`[settings.lockr]`, default key `id5id`). A cookie holding the ID5 fetch response JSON //! (plain or percent-encoded) is parsed exactly like a lookup response, and //! a bare ID string becomes one UID on the first source with agent type 1 //! and the default TTL. The response `signature` is never stored. When a @@ -46,7 +47,8 @@ use url::Url; use serde_json::value::RawValue; use crate::common::{ - VendorEid, bounded_expiry, is_routable_ip, is_usable_id, join_section_ids, percent_decoded, + CookieReader, LockrSettings, ReadsLockr, VendorEid, bounded_expiry, is_routable_ip, + is_usable_id, join_section_ids, percent_decoded, }; /// The UID ID5 returns for a visitor who opted out. @@ -71,6 +73,9 @@ pub struct Id5Settings { /// yielding a usable UID wins. #[serde(default = "Id5Settings::default_cookie_names")] pub cookie_names: Vec, + /// The Lockr capability; see [`LockrSettings`]. + #[serde(default)] + pub lockr: LockrSettings, } impl Id5Settings { @@ -85,7 +90,11 @@ pub struct Id5Provider { capabilities: ProviderCapabilities, url: Url, partner_id: u32, - cookie_names: Box<[String]>, + cookies: CookieReader, +} + +impl ReadsLockr for Id5 { + const LOCKR_COOKIE_NAMES: &'static [&'static str] = &["id5id"]; } impl ProviderKind for Id5 { @@ -131,7 +140,11 @@ impl ProviderKind for Id5 { }, url, partner_id: settings.partner_id, - cookie_names: settings.cookie_names.into_boxed_slice(), + cookies: CookieReader::with_lockr::( + common, + settings.cookie_names, + settings.lockr, + )?, }) } } @@ -352,10 +365,7 @@ impl IdentityProvider for Id5Provider { &self, cookies: RequestCookies<'cookie>, ) -> Option>> { - self.cookie_names - .iter() - .filter_map(|name| cookies.get(name)) - .find_map(|value| self.cookie_eids(value)) + self.cookies.read(cookies, |value| self.cookie_eids(value)) } } @@ -733,6 +743,43 @@ mod tests { ); } + #[test] + fn reads_lockr_after_own_cookies_and_falls_back_to_the_lookup() { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "id5", + "kind": "id5", + "endpoint": "https://id5.example.com/g/v2", + "allowed_domains": ["id5.example.com"], + "sources": ["id5.example.com"], + "settings": { + "partner_id": 1234, + "lockr": { "enabled": true, "cookie_names": ["example_lockr_id5"] }, + }, + })) + .expect("should parse provider config"); + let provider = + build_kind::(&config, &ProviderSecrets::default()).expect("should build provider"); + + let own_first = read( + &provider, + "example_lockr_id5=lockr-example; id5id=own-example", + ); + let lockr = read(&provider, "id5id=0; example_lockr_id5=lockr-example"); + let neither = read(&provider, "example_lockr_id5=0"); + + assert_eq!( + own_first.map(|uids| uids[0].1.clone()).as_deref(), + Some("own-example"), + "should prefer the own cookie" + ); + assert_eq!( + lockr.map(|uids| uids[0].1.clone()).as_deref(), + Some("lockr-example"), + "should read the Lockr cookie when the own cookie yields nothing" + ); + assert!(neither.is_none(), "should leave the ID to the lookup"); + } + #[test] fn ignores_opted_out_and_malformed_cookies_so_the_lookup_runs() { let provider = provider(); diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index d8eca7bab..37b2539f2 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -10,6 +10,8 @@ //! depends on it. mod common; + +pub use common::LockrSettings; pub mod connectid; pub mod id5; pub mod liveintent; @@ -44,6 +46,19 @@ pub fn is_registered_kind(kind: &str) -> bool { ) } +/// Whether `kind` declares the Lockr capability, so its settings accept a +/// `[ec.identity_providers.settings.lockr]` table. +/// +/// The settings of every other kind reject that table as an unknown field +/// when the provider is built. +#[must_use] +pub fn reads_lockr(kind: &str) -> bool { + matches!( + kind, + id5::Id5::KIND | liveramp::LiveRamp::KIND | uid2::Uid2::KIND + ) +} + /// Builds the partner registry, marking the sources claimed by enabled /// providers as provider-owned. /// @@ -162,6 +177,8 @@ fn build_phase_coordinator( #[cfg(test)] mod tests { + use serde_json::json; + use super::*; fn create_test_settings() -> Settings { @@ -185,6 +202,72 @@ mod tests { .expect("should parse test settings") } + #[test] + fn only_lockr_capable_kinds_accept_lockr_settings() { + for (kind, endpoint, sources, settings) in [ + ( + "id5", + Some("https://id5.example.com/g/v2"), + json!(["id5.example.com"]), + json!({ "partner_id": 1234 }), + ), + ( + "liveintent", + Some("https://li.example.com/idex"), + json!(["li.example.com"]), + json!({ "attributes": { "nonId": "li.example.com" } }), + ), + ("liveramp", None, json!(["liveramp.example.com"]), json!({})), + ( + "connectid", + None, + json!(["connectid.example.com"]), + json!({}), + ), + ("uid2", None, json!(["uid2.example.com"]), json!({})), + ] { + let mut settings = settings; + settings["lockr"] = json!({ "enabled": true }); + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": kind, + "kind": kind, + "endpoint": endpoint, + "allowed_domains": ["id5.example.com", "li.example.com"], + "sources": sources, + "settings": settings, + })) + .expect("should parse provider config"); + + let built = build(&config, &ProviderSecrets::default()); + + assert_eq!( + built.is_ok(), + reads_lockr(kind), + "should accept lockr settings only for capable kind {kind}" + ); + } + } + + #[test] + fn enabled_lockr_rejects_an_empty_cookie_list() { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "uid2", + "kind": "uid2", + "sources": ["uid2.example.com"], + "settings": { "lockr": { "enabled": true, "cookie_names": [] } }, + })) + .expect("should parse provider config"); + + let error = build(&config, &ProviderSecrets::default()) + .err() + .expect("should reject an empty lockr cookie list"); + + assert!( + matches!(error.current_context(), ProviderConfigError::Invalid), + "should report invalid config" + ); + } + #[test] fn unknown_kind_is_rejected() { let config: IdentityProviderConfig = toml::from_str( diff --git a/crates/trusted-server-identity-providers/src/liveintent.rs b/crates/trusted-server-identity-providers/src/liveintent.rs index b61ea9226..6139fbe05 100644 --- a/crates/trusted-server-identity-providers/src/liveintent.rs +++ b/crates/trusted-server-identity-providers/src/liveintent.rs @@ -4,6 +4,9 @@ //! and asks for the attributes the operator configured. Each attribute maps //! to one configured `OpenRTB` source. The lookup is anonymous: no hashed //! email and no first-party `LiveIntent` cookie are sent. +//! +//! This kind has no Lockr capability: Lockr AIM is not documented to +//! deliver `LiveIntent` IDs, so a `lockr` settings table is rejected. use std::borrow::Cow; use std::collections::{BTreeMap, HashMap}; diff --git a/crates/trusted-server-identity-providers/src/liveramp.rs b/crates/trusted-server-identity-providers/src/liveramp.rs index 3a091ca3e..384474002 100644 --- a/crates/trusted-server-identity-providers/src/liveramp.rs +++ b/crates/trusted-server-identity-providers/src/liveramp.rs @@ -7,8 +7,9 @@ //! [`ENVELOPE_TTL`]. No server call is made: the refresh contract is not //! confirmed, so the browser keeps refreshing the cookie. //! -//! Lockr AIM can write the same envelope into a cookie whose name comes -//! from its own configuration; list that name in `cookie_names`. +//! With the Lockr capability enabled (`[settings.lockr]`), the cookies Lockr +//! AIM writes the envelope into (default key `_lr_env`) are read after the +//! own cookies, with the same parser. //! //! Parsing is lenient: an undecodable cookie, a body without a string //! `envelope`, or a blank or opt-out envelope reads as no ID. @@ -28,7 +29,8 @@ use trusted_server_core::ec::provider::{ }; use crate::common::{ - bounded_expiry, cookie_capabilities, cookie_eid, is_usable_id, percent_decoded, + CookieReader, LockrSettings, ReadsLockr, bounded_expiry, cookie_capabilities, cookie_eid, + is_usable_id, percent_decoded, }; /// How long a stored envelope is kept. @@ -44,6 +46,9 @@ pub struct LiveRampSettings { /// Cookies read in order; the first holding an envelope wins. #[serde(default = "LiveRampSettings::default_cookie_names")] pub cookie_names: Vec, + /// The Lockr capability; see [`LockrSettings`]. + #[serde(default)] + pub lockr: LockrSettings, } impl LiveRampSettings { @@ -57,7 +62,11 @@ impl LiveRampSettings { pub struct LiveRampProvider { capabilities: ProviderCapabilities, source: SourceDomain, - cookie_names: Box<[String]>, + cookies: CookieReader, +} + +impl ReadsLockr for LiveRamp { + const LOCKR_COOKIE_NAMES: &'static [&'static str] = &["_lr_env"]; } impl ProviderKind for LiveRamp { @@ -69,7 +78,9 @@ impl ProviderKind for LiveRamp { common: &ProviderCommon<'_>, settings: LiveRampSettings, ) -> Result> { - if settings.cookie_names.is_empty() { + let cookies = + CookieReader::with_lockr::(common, settings.cookie_names, settings.lockr)?; + if cookies.is_empty() { return Err(Report::new(ProviderConfigError::Invalid) .attach("a liveramp provider needs at least one cookie name")); } @@ -77,7 +88,7 @@ impl ProviderKind for LiveRamp { Ok(LiveRampProvider { capabilities, source, - cookie_names: settings.cookie_names.into_boxed_slice(), + cookies, }) } } @@ -121,11 +132,7 @@ impl IdentityProvider for LiveRampProvider { &self, cookies: RequestCookies<'cookie>, ) -> Option>> { - let envelope = self - .cookie_names - .iter() - .filter_map(|name| cookies.get(name)) - .find_map(envelope)?; + let envelope = self.cookies.read(cookies, envelope)?; Some(vec![cookie_eid( &self.source, Cow::Owned(envelope), @@ -209,6 +216,47 @@ mod tests { ); } + #[test] + fn ignores_lockr_cookies_while_lockr_is_disabled() { + let provider = provider(&json!({ + "lockr": { "cookie_names": ["example_lockr_env"] }, + })); + let value = encoded(&json!({ "envelope": "env-example" }), &STANDARD); + + let read = read(&*provider, &format!("example_lockr_env={value}")); + + assert!( + read.is_none(), + "should not read a Lockr cookie when disabled" + ); + } + + #[test] + fn reads_a_custom_lockr_cookie_after_the_own_cookie() { + let provider = provider(&json!({ + "lockr": { "enabled": true, "cookie_names": ["example_lockr_env"] }, + })); + let own = encoded(&json!({ "envelope": "env-own" }), &STANDARD); + let lockr = encoded(&json!({ "envelope": "env-lockr" }), &STANDARD); + + let both = read( + &*provider, + &format!("example_lockr_env={lockr}; _lr_env={own}"), + ); + let lockr_only = read(&*provider, &format!("example_lockr_env={lockr}")); + + assert_eq!( + both.map(|(id, _)| id).as_deref(), + Some("env-own"), + "should prefer the own cookie" + ); + assert_eq!( + lockr_only.map(|(id, _)| id).as_deref(), + Some("env-lockr"), + "should fall back to the Lockr cookie" + ); + } + #[test] fn ignores_missing_malformed_and_opt_out_cookies() { let provider = provider(&json!({})); diff --git a/crates/trusted-server-identity-providers/src/uid2.rs b/crates/trusted-server-identity-providers/src/uid2.rs index d010337dd..6f0a137ac 100644 --- a/crates/trusted-server-identity-providers/src/uid2.rs +++ b/crates/trusted-server-identity-providers/src/uid2.rs @@ -8,10 +8,12 @@ //! passed. The refresh token and response key are never read, so they are //! never stored, and no refresh is made server side. //! -//! When `__uid_2` yields nothing, Prebid's `__uid2_advertising_token` -//! cookie (or each name in `cookie_names`) is read as a raw token with the shared default TTL. Lockr AIM can -//! write the token into a cookie whose name comes from its own -//! configuration; list that name in `cookie_names`. +//! When `__uid_2` yields nothing, the cookies in `cookie_names` (default +//! Prebid's `__uid2_advertising_token`) are read as a raw token with the +//! shared default TTL. With the Lockr capability enabled +//! (`[settings.lockr]`), the cookies Lockr AIM writes the raw token into +//! (default key `__uid2_advertising_token`) are read last, with the same +//! parser. //! //! Parsing is lenient: an undecodable cookie, a missing token or expiry, //! or a blank or opt-out token reads as no ID. @@ -28,7 +30,8 @@ use trusted_server_core::ec::provider::{ }; use crate::common::{ - bounded_expiry, cookie_capabilities, cookie_eid, is_usable_id, percent_decoded, + CookieReader, LockrSettings, ReadsLockr, bounded_expiry, cookie_capabilities, cookie_eid, + is_usable_id, percent_decoded, }; /// The UID2 provider kind. @@ -45,6 +48,9 @@ pub struct Uid2Settings { /// identity cookie. #[serde(default = "Uid2Settings::default_cookie_names")] pub cookie_names: Vec, + /// The Lockr capability; see [`LockrSettings`]. + #[serde(default)] + pub lockr: LockrSettings, } impl Uid2Settings { @@ -63,7 +69,11 @@ pub struct Uid2Provider { capabilities: ProviderCapabilities, source: SourceDomain, identity_cookie_name: String, - cookie_names: Box<[String]>, + cookies: CookieReader, +} + +impl ReadsLockr for Uid2 { + const LOCKR_COOKIE_NAMES: &'static [&'static str] = &["__uid2_advertising_token"]; } impl ProviderKind for Uid2 { @@ -80,7 +90,11 @@ impl ProviderKind for Uid2 { capabilities, source, identity_cookie_name: settings.identity_cookie_name, - cookie_names: settings.cookie_names.into_boxed_slice(), + cookies: CookieReader::with_lockr::( + common, + settings.cookie_names, + settings.lockr, + )?, }) } } @@ -136,12 +150,7 @@ impl IdentityProvider for Uid2Provider { let (token, expiry) = cookies .get(&self.identity_cookie_name) .and_then(identity) - .or_else(|| { - self.cookie_names - .iter() - .filter_map(|name| cookies.get(name)) - .find_map(raw_token) - })?; + .or_else(|| self.cookies.read(cookies, raw_token))?; Some(vec![cookie_eid(&self.source, Cow::Owned(token), expiry)]) } } @@ -226,6 +235,36 @@ mod tests { ); } + #[test] + fn reads_the_lockr_cookie_last() { + let enabled = provider(&json!({ + "lockr": { "enabled": true, "cookie_names": ["example_lockr_uid2"] }, + })); + let disabled = provider(&json!({ + "lockr": { "cookie_names": ["example_lockr_uid2"] }, + })); + let own = identity_cookie(&json!({ + "advertising_token": "AgAAAAowntoken", + "identity_expires": 1_700_000_000_000_u64, + })); + + let both = read(&*enabled, &format!("example_lockr_uid2=AgAAAAlockr; {own}")); + let lockr_only = read(&*enabled, "example_lockr_uid2=AgAAAAlockr"); + let ignored = read(&*disabled, "example_lockr_uid2=AgAAAAlockr"); + + assert_eq!( + both.map(|(token, _)| token).as_deref(), + Some("AgAAAAowntoken"), + "should prefer the identity cookie" + ); + assert_eq!( + lockr_only.map(|(token, _)| token).as_deref(), + Some("AgAAAAlockr"), + "should read the Lockr cookie as a raw token" + ); + assert!(ignored.is_none(), "should ignore Lockr while disabled"); + } + #[test] fn ignores_missing_malformed_and_opt_out_cookies() { let provider = provider(&json!({})); diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-connectid.md b/docs/superpowers/plans/2026-11-02-identity-provider-connectid.md index 05404d501..c10e2dbf1 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-connectid.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-connectid.md @@ -15,6 +15,8 @@ - Only `connectId`, `lastSynced` and `ttl` are read. `he` and `puid` are never deserialized, so they are never stored. - A missing `connectId` or `lastSynced`, a non-string ID, a blank or opt-out ID, or malformed JSON reads as no ID. +No Lockr capability: Lockr AIM is not documented to deliver ConnectID, so a `[settings.lockr]` table is rejected. + ## Stored ID - One EID for the single configured source, one UID with `atype = 3`. diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-id5.md b/docs/superpowers/plans/2026-11-02-identity-provider-id5.md index 81eb5c323..77d5f84c2 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-id5.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-id5.md @@ -31,9 +31,10 @@ ## First-party cookie -ID5's source is provider-owned, so the browser `ts-eids` copy is dropped. Prebid's ID5 module and Lockr AIM store the ID in a first-party cookie instead, which `read_request_cookies` reads before any lookup. +ID5's source is provider-owned, so the browser `ts-eids` copy is dropped. Prebid's ID5 module stores the ID in a first-party cookie instead, which `read_request_cookies` reads before any lookup. -- Setting `cookie_names` (default `["id5id"]`), tried in order; Lockr's key name is configurable, so operators list it. +- Setting `cookie_names` (default `["id5id"]`), tried in order. +- `id5` declares the Lockr capability (see `2026-11-02-identity-provider-lockr-capability.md`): with `[settings.lockr] enabled = true`, the Lockr cookies (default `["id5id"]`) are read after `cookie_names` and before the lookup, with the same parser. - A value that is JSON after percent-decoding is parsed with the same code as a lookup response (lenient entries, opt-out `0` dropped, normalized sources, `cache_control.max_age_sec` bounded). `ext` objects are dropped when the value was percent-encoded, since they borrow from the decoded copy. - Any other value is a bare ID: one UID on the first source with `atype = 1` and the default TTL. - `signature` is never stored as a UID. A cookie that yields no usable UID is ignored, and the lookup runs as usual; a kept UID skips the lookup. diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md b/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md index 94d1026cf..3bd288357 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md @@ -24,6 +24,8 @@ | `distributor_id` | none | Sent as `did`; the path uses `prebid` when unset. | | `attributes` | required, ≥ 1 | `LiveIntent` attribute name → stored source. | +No Lockr capability: Lockr AIM is not documented to deliver LiveIntent IDs, and the lookup uses LiveIntent's own signals, so a `[settings.lockr]` table is rejected. + ## Request - `GET //` with `Origin: https://`. diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md b/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md index c91ce6c50..cbc581a31 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md @@ -22,7 +22,7 @@ ## Lockr -Lockr AIM writes the LiveRamp envelope into a cookie named by its own configuration. List that name in `cookie_names`; no separate Lockr kind is needed. +`liveramp` declares the Lockr capability (see `2026-11-02-identity-provider-lockr-capability.md`). With `[settings.lockr] enabled = true`, the Lockr cookies (default `["_lr_env"]`) are read after `cookie_names` with the same envelope parser. `cookie_names` may be empty only when Lockr is enabled. ## Tasks diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-lockr-capability.md b/docs/superpowers/plans/2026-11-02-identity-provider-lockr-capability.md new file mode 100644 index 000000000..ca4949cae --- /dev/null +++ b/docs/superpowers/plans/2026-11-02-identity-provider-lockr-capability.md @@ -0,0 +1,24 @@ +# Lockr Cookie Capability Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Read vendor IDs that Lockr AIM writes into first-party cookies on the publisher domain, as an explicit, reusable capability of the provider kinds Lockr delivers (ID5, LiveRamp, UID2), instead of ad-hoc cookie names per provider. + +**Architecture:** Everything lives in `crates/trusted-server-identity-providers/src/common.rs`; core only passes the configured provider ID to builds (`ProviderCommon::id`) for logging. + +- `LockrSettings` (`enabled`, default `false`; `cookie_names`, default the vendor's known Lockr key), parsed from `[ec.identity_providers.settings.lockr]` as a field of each capable kind's settings. +- `ReadsLockr`: a trait a `ProviderKind` implements to declare the capability, with `LOCKR_COOKIE_NAMES` (`id5id` for ID5, `_lr_env` for LiveRamp, `__uid2_advertising_token` for UID2). `reads_lockr(kind)` in `lib.rs` lists the same kinds. +- `CookieReader`: holds the own cookie names and, when Lockr is enabled, the Lockr names not already among them. `read` hands each value to the provider's existing parser: own cookies first, then Lockr cookies. Lookup kinds look up only when neither yields an ID. +- A read from a Lockr cookie logs at debug with the provider ID, the cookie name and origin `lockr`. + +**Gating:** kinds without the capability (ConnectID, LiveIntent) have `deny_unknown_fields` settings without a `lockr` field, so a `lockr` table fails the provider build at startup. Lockr enabled with an empty `cookie_names` is rejected. + +**Own-cookie settings:** every cookie-reading kind takes a `cookie_names` list (UID2's `token_cookie_name` and ConnectID's `cookie_name` are renamed). Lockr names move out of them into `[settings.lockr]`. + +## Tasks + +- [ ] Add `ProviderCommon::id` in core and set it in `build_kind`. +- [ ] Rename UID2 `token_cookie_name` and ConnectID `cookie_name` to `cookie_names`. +- [ ] Add `LockrSettings`, `ReadsLockr` and `CookieReader` to `common.rs`; wire ID5, LiveRamp and UID2; use `CookieReader::own` for ConnectID. +- [ ] Tests: capability gating per kind, empty Lockr list rejected, Lockr disabled ignores the cookie, a custom Lockr name is read, own cookie before Lockr, no cookie leaves ID5 to the lookup. +- [ ] Update `trusted-server.example.toml` and the provider plans. diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md b/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md index 6a6389571..b09c3f2bc 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md @@ -23,7 +23,7 @@ ## Lockr -Lockr AIM writes the UID2 token into a cookie named by its own configuration. List that name in `cookie_names`. Lockr-delivered LiveRamp envelopes are handled by the `liveramp` kind's `cookie_names`, so no separate Lockr kind exists. Lockr-delivered ID5 cookies are not read; the `id5` kind resolves ID5 by lookup. +`uid2` declares the Lockr capability (see `2026-11-02-identity-provider-lockr-capability.md`). With `[settings.lockr] enabled = true`, the Lockr cookies (default `["__uid2_advertising_token"]`) are read last, after the identity cookie and `cookie_names`, with the raw-token parser. Lockr names already in `cookie_names` are not read twice. ## Tasks diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 573b07890..54047851e 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -154,8 +154,14 @@ pull_sync_concurrency = 3 # ID5 (kind "id5"): anonymous lookup, no secrets. The first source receives # the legacy `universal_uid`. Use the endpoint and sources ID5 gives you. # Before any lookup it reads the first-party cookies in `cookie_names`, in -# order (Prebid's ID5 module writes `id5id`; Lockr AIM writes the ID under a -# key name from its own configuration, so list that name too). +# order (Prebid's ID5 module writes `id5id`). +# +# Lockr capability (id5, liveramp and uid2 only): Lockr AIM writes other +# vendors' IDs into first-party cookies under key names configured per +# publisher in Lockr. Enable `[ec.identity_providers.settings.lockr]` to read +# them after the vendor's own cookies and before any lookup; `cookie_names` +# defaults to the vendor's known Lockr key. connectid and liveintent reject a +# `lockr` table. # [[ec.identity_providers]] # id = "id5" # kind = "id5" @@ -167,6 +173,9 @@ pull_sync_concurrency = 3 # [ec.identity_providers.settings] # partner_id = 1234 # cookie_names = ["id5id"] +# [ec.identity_providers.settings.lockr] +# enabled = true +# cookie_names = ["id5id"] # # LiveIntent (kind "liveintent"): anonymous lookup, no secrets. Each attribute # LiveIntent returns is stored under the source it maps to; every mapped @@ -187,8 +196,8 @@ pull_sync_concurrency = 3 # # LiveRamp (kind "liveramp"): no lookup and no endpoint. Reads the RampID # envelope from LiveRamp's own first-party `_lr_env` cookie and keeps it for -# 15 days. Takes exactly one source; use the one LiveRamp gives you. Add a -# cookie name that Lockr AIM writes the envelope into to `cookie_names`. +# 15 days. Takes exactly one source; use the one LiveRamp gives you. Supports +# the Lockr capability. # [[ec.identity_providers]] # id = "liveramp" # kind = "liveramp" @@ -197,6 +206,9 @@ pull_sync_concurrency = 3 # tcf_vendor_id = 9999 # [ec.identity_providers.settings] # cookie_names = ["_lr_env"] +# [ec.identity_providers.settings.lockr] +# enabled = true +# cookie_names = ["_lr_env"] # # Yahoo ConnectID (kind "connectid"): no lookup and no endpoint. Reads # ConnectID from Yahoo's own first-party `connectId` cookie and keeps it until @@ -214,7 +226,8 @@ pull_sync_concurrency = 3 # from the UID2 SDK's `__uid_2` cookie until its `identity_expires`, falling # back to Prebid's raw `__uid2_advertising_token` cookie. Never stores or uses # the refresh token. Takes exactly one source. `cookie_names` lists the raw -# token cookies, read in order after `identity_cookie_name`. +# token cookies, read in order after `identity_cookie_name`. Supports the +# Lockr capability; Lockr cookies are read last, as raw tokens. # [[ec.identity_providers]] # id = "uid2" # kind = "uid2" @@ -223,6 +236,9 @@ pull_sync_concurrency = 3 # [ec.identity_providers.settings] # identity_cookie_name = "__uid_2" # cookie_names = ["__uid2_advertising_token"] +# [ec.identity_providers.settings.lockr] +# enabled = true +# cookie_names = ["__uid2_advertising_token"] # ============================================================================= From 7b0e08554e7f0705981eddac9ae0ed7c143359ea Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 16:59:14 +0530 Subject: [PATCH 12/72] Mark LiveIntent partner IDs and require a did- distributor ID Partner attributes such as bidswitch or magnite now carry the UID ext {provider: liveintent.com} that Prebid's LiveIntent module sets, while nonId stays bare. LiveIntent answers 400 Malformed did to any distributor ID that is not a did- ID, so the build rejects one. Signed-off-by: dhruv8sh --- .../src/liveintent.rs | 107 +++++++++++++++++- ...2026-11-02-identity-provider-liveintent.md | 14 +-- trusted-server.example.toml | 12 +- 3 files changed, 121 insertions(+), 12 deletions(-) diff --git a/crates/trusted-server-identity-providers/src/liveintent.rs b/crates/trusted-server-identity-providers/src/liveintent.rs index 6139fbe05..ae17f15ba 100644 --- a/crates/trusted-server-identity-providers/src/liveintent.rs +++ b/crates/trusted-server-identity-providers/src/liveintent.rs @@ -5,6 +5,9 @@ //! to one configured `OpenRTB` source. The lookup is anonymous: no hashed //! email and no first-party `LiveIntent` cookie are sent. //! +//! As in Prebid's `LiveIntent` module, every attribute other than `nonId` is +//! a partner ID and its UID carries `ext.provider` naming `LiveIntent`. +//! //! This kind has no Lockr capability: Lockr AIM is not documented to //! deliver `LiveIntent` IDs, so a `lockr` settings table is rejected. @@ -34,6 +37,15 @@ use crate::common::{bounded_expiry, is_routable_ip, join_section_ids}; /// `atype` `LiveIntent` IDs carry: a person-based ID. const LIVEINTENT_ATYPE: i32 = 3; +/// The attribute holding `LiveIntent`'s own ID; the others are partner IDs. +const NON_ID_ATTRIBUTE: &str = "nonId"; + +/// UID `ext` Prebid puts on partner IDs that `LiveIntent` resolved. +const PARTNER_UID_EXT: &str = r#"{"provider":"liveintent.com"}"#; + +/// Prefix of a `LiveIntent`-issued distributor ID, such as `did-0123`. +const DISTRIBUTOR_ID_PREFIX: &str = "did-"; + /// Most UIDs accepted per `LiveIntent` EID. const MAX_UIDS_PER_EID: NonZeroUsize = NonZeroUsize::new(1).expect("should be non-zero"); @@ -47,7 +59,8 @@ pub struct LiveIntentSettings { /// The `LiveIntent` publisher ID; `LiveIntent` accepts `any`. #[serde(default = "default_publisher_id")] pub publisher_id: String, - /// The distributor ID, when the publisher has one. + /// The `LiveIntent`-issued distributor ID, such as `did-0123`, when the + /// publisher has one. #[serde(default)] pub distributor_id: Option, /// Map of `LiveIntent` attribute name to the source it is stored under. @@ -67,6 +80,15 @@ pub struct LiveIntentProvider { url: Url, distributor_id: Option, attributes: Box<[(String, SourceDomain)]>, + partner_uid_ext: &'static RawValue, +} + +/// Whether `id` has the `did-` form `LiveIntent` accepts; it answers any +/// other distributor ID with `400 Malformed did`. +fn is_distributor_id(id: &str) -> bool { + id.strip_prefix(DISTRIBUTOR_ID_PREFIX).is_some_and(|rest| { + !rest.is_empty() && rest.bytes().all(|byte| byte.is_ascii_alphanumeric()) + }) } impl ProviderKind for LiveIntent { @@ -98,6 +120,17 @@ impl ProviderKind for LiveIntent { Report::new(ProviderConfigError::Invalid) .attach("a liveintent provider needs at least one attribute") })?; + if let Some(id) = settings + .distributor_id + .as_deref() + .filter(|id| !is_distributor_id(id)) + { + return Err(Report::new(ProviderConfigError::Invalid).attach(format!( + "distributor_id `{id}` is not a LiveIntent `did-` distributor ID" + ))); + } + let partner_uid_ext = serde_json::from_str::<&'static RawValue>(PARTNER_UID_EXT) + .change_context(ProviderConfigError::Invalid)?; let distributor = settings.distributor_id.as_deref().unwrap_or("prebid"); let mut url = common .endpoint @@ -131,6 +164,7 @@ impl ProviderKind for LiveIntent { url, distributor_id: settings.distributor_id, attributes, + partner_uid_ext, }) } } @@ -238,7 +272,7 @@ impl IdentityProvider for LiveIntentProvider { uids: vec![ProviderUid { id, atype: Some(LIVEINTENT_ATYPE), - ext: None, + ext: (attribute != NON_ID_ATTRIBUTE).then_some(self.partner_uid_ext), expiry, }], }) @@ -407,10 +441,79 @@ mod tests { matches!(eids[0].uids[0].expiry, ProviderExpiry::After(d) if d == Duration::from_secs(7200)), "should use Expires relative to Date" ); + assert!( + eids[0].uids[0].ext.is_none(), + "should keep nonId without ext" + ); assert_eq!( eids[1].source, "li-partner.example.com", "should map partner" ); + assert_eq!( + eids[1].uids[0].ext.map(RawValue::get), + Some(r#"{"provider":"liveintent.com"}"#), + "should name LiveIntent as the provider of a partner ID" + ); + } + + #[test] + fn rejects_a_distributor_id_liveintent_did_not_issue() { + for distributor_id in ["prebid", "did-", "did-01/2"] { + let mut config = config(&json!({ "nonId": "li.example.com" })); + config + .settings + .insert("distributor_id".to_owned(), json!(distributor_id)); + + let error = build_kind::(&config, &ProviderSecrets::default()) + .err() + .expect("should reject the distributor ID"); + + assert!( + matches!(error.current_context(), ProviderConfigError::Invalid), + "should report invalid config for `{distributor_id}`" + ); + } + } + + #[test] + fn calls_the_distributor_path_with_a_did_distributor_id() { + let mut config = config(&json!({ "nonId": "li.example.com" })); + config + .settings + .insert("distributor_id".to_owned(), json!("did-0123")); + let provider = build_kind::(&config, &ProviderSecrets::default()) + .expect("should build provider"); + let consent = ConsentContext::default(); + let due = due(); + let lookup = LookupInput { + ec_id: "ec-example", + consent: &consent, + client: ClientSignals { + ip: None, + user_agent: None, + }, + publisher_domain: "www.example.com", + due_sources: &due, + current: OwnedEntries::new(None, &[]), + hashed_email: None, + }; + + let request = provider + .build_request(&lookup) + .expect("should build request"); + + assert_eq!( + request.url.path(), + "/idex/did-0123/12345", + "should call the distributor path" + ); + assert!( + request + .url + .query_pairs() + .any(|(key, value)| key == "did" && value == "did-0123"), + "should send the distributor ID" + ); } #[test] diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md b/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md index 3bd288357..6918d4e23 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md @@ -13,16 +13,16 @@ - Anonymous only: no hashed email, and no first-party `LiveIntent` cookie (`duid`), because `LookupInput` carries no cookies. - The attribute-to-source table is operator configuration (`settings.attributes`), not a hard-coded list; every mapped source must be one of the provider's `sources`. - No page URL in `LookupInput`, so `pu` is the publisher origin. -- EID-level fields are ignored and no `ext` is synthesized (`segments`, `provider`, `rtiPartner`), because `ProviderUid::ext` borrows from the response body. +- EID-level fields are ignored. As in Prebid, every attribute other than `nonId` is a partner ID whose UID carries a static `ext` of `{"provider":"liveintent.com"}`; no other `ext` (`segments`, `rtiPartner`) is synthesized. - The `Expires` header becomes `ProviderExpiry::At`; the provider never reads the clock. ## Settings -| Key | Default | Meaning | -| ---------------- | ------------- | ------------------------------------------------- | -| `publisher_id` | `any` | Path segment after the distributor. | -| `distributor_id` | none | Sent as `did`; the path uses `prebid` when unset. | -| `attributes` | required, ≥ 1 | `LiveIntent` attribute name → stored source. | +| Key | Default | Meaning | +| ---------------- | ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| `publisher_id` | `any` | Path segment after the distributor. | +| `distributor_id` | none | A `did-...` ID; sent as `did` and used as the path. LiveIntent rejects any other form, so build rejects it. The path uses `prebid` when unset. | +| `attributes` | required, ≥ 1 | `LiveIntent` attribute name → stored source. | No Lockr capability: Lockr AIM is not documented to deliver LiveIntent IDs, and the lookup uses LiveIntent's own signals, so a `[settings.lockr]` table is rejected. @@ -33,7 +33,7 @@ No Lockr capability: Lockr AIM is not documented to deliver LiveIntent IDs, and ## Response -- `204 No Content` is `LookupOutcome::NoMatch`; another non-2xx status or a non-object body is `ProviderError::InvalidResponse`. +- `204 No Content` is `LookupOutcome::NoMatch`. LiveIntent answers `204` to anonymous `gdpr=1` lookups and returns only `nonId` without its own third-party cookie; another non-2xx status or a non-object body is `ProviderError::InvalidResponse`. - Each configured attribute with a non-empty string value becomes one EID with `atype = 3`, borrowed from the body. ## Tasks diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 54047851e..4c3e26bd6 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -179,20 +179,26 @@ pull_sync_concurrency = 3 # # LiveIntent (kind "liveintent"): anonymous lookup, no secrets. Each attribute # LiveIntent returns is stored under the source it maps to; every mapped -# source must be listed in `sources`. +# source must be listed in `sources`. `nonId` is LiveIntent's own ID; the other +# attributes (Prebid's names, such as `bidswitch`, `triplelift`, `magnite`, +# `index`, `openx`, `pubmatic`, `sovrn` and `nexxen`) are partner IDs, sent +# with UID `ext.provider` naming LiveIntent. `distributor_id` is optional and +# must be a LiveIntent-issued `did-...` ID; without it the lookup uses the +# `prebid` path, as Prebid does. # [[ec.identity_providers]] # id = "liveintent" # kind = "liveintent" # enabled = false # endpoint = "https://liveintent.example.com/idex" # allowed_domains = ["liveintent.example.com"] -# sources = ["liveintent.example.com"] +# sources = ["liveintent.example.com", "ssp.example.com"] # tcf_vendor_id = 9999 # [ec.identity_providers.settings] # publisher_id = "any" -# distributor_id = "example-distributor" +# distributor_id = "did-0123" # [ec.identity_providers.settings.attributes] # nonId = "liveintent.example.com" +# magnite = "ssp.example.com" # # LiveRamp (kind "liveramp"): no lookup and no endpoint. Reads the RampID # envelope from LiveRamp's own first-party `_lr_env` cookie and keeps it for From 75a9477af53ee377274c2bcac5d6fd6375c4c718 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 19:12:25 +0530 Subject: [PATCH 13/72] Let identity providers build an EID ext and read cookies in lookups Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 7 ++--- .../src/ec/coordinator/validate.rs | 27 ++++++++++++++++--- crates/trusted-server-core/src/ec/provider.rs | 11 +++++--- .../src/common.rs | 4 +-- .../src/id5.rs | 2 ++ .../src/liveintent.rs | 6 ++++- .../src/liveramp.rs | 1 + 7 files changed, 45 insertions(+), 13 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 5641a0d5b..d85359da5 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -40,9 +40,9 @@ use crate::settings::IdentityProviderConfig; /// Coordinates server-side identity lookups for one navigation. /// /// Implementations own eligibility, consent, destination checks, the shared -/// time budget and validation. Providers never see KV, cookies, HTTP or the -/// clock. Implementations never write KV: they return the change and the -/// caller spends the [`crate::ec::page_load::PageLoadWrite`] permit. +/// time budget and validation. Providers never see KV, HTTP or the clock, +/// and see cookies only as [`RequestCookies`]. Implementations never write +/// KV: they return the change and the caller spends the [`crate::ec::page_load::PageLoadWrite`] permit. #[allow(async_fn_in_trait)] pub trait IdentityCoordinator { /// Starts lookups for every eligible provider without waiting. @@ -550,6 +550,7 @@ impl IdentityCoordinator for ProviderCoordinator

{ publisher_domain: lookup.publisher_domain, due_sources: &due, current: OwnedEntries::new(entry, ®istered.policy.sources, ®istered.id), + cookies: lookup.cookies, hashed_email: None, }; if let Some(mut eids) = registered.provider.generate(&input) { diff --git a/crates/trusted-server-core/src/ec/coordinator/validate.rs b/crates/trusted-server-core/src/ec/coordinator/validate.rs index 9d9a63b17..2f08be0ee 100644 --- a/crates/trusted-server-core/src/ec/coordinator/validate.rs +++ b/crates/trusted-server-core/src/ec/coordinator/validate.rs @@ -117,7 +117,7 @@ fn eid_fields( inserter: bounded_field(eid.inserter.as_ref())?, matcher: bounded_field(eid.matcher.as_ref())?, mm: eid.mm.map(MatchMethod::try_from).transpose()?, - ext: eid.ext.map(RawJson::try_from).transpose()?, + ext: eid.ext.as_deref().map(RawJson::try_from).transpose()?, }) } @@ -128,7 +128,7 @@ fn stored_uid( ) -> Result { let id = UidValue::try_from(uid.id.into_owned())?; let atype = AgentType::try_from(uid.atype.unwrap_or(default_atype))?; - let ext = uid.ext.map(RawJson::try_from).transpose()?; + let ext = uid.ext.as_deref().map(RawJson::try_from).transpose()?; let expiry = match resolve_expiry(uid.expiry, policy.now) { Some(at) => Expiry::At(at), None => Expiry::Never, @@ -326,7 +326,7 @@ mod tests { expired.expiry = ProviderExpiry::At(UnixSeconds::new(1000)); let array_ext = RawValue::from_string("[1]".to_owned()).expect("should build raw"); let mut bad_ext = uid("u-ext"); - bad_ext.ext = Some(&array_ext); + bad_ext.ext = Some(Cow::Borrowed(&array_ext)); let (entries, dropped) = validate( vec![ @@ -434,4 +434,25 @@ mod tests { "should keep valid mm" ); } + + #[test] + fn preserves_an_eid_ext_the_provider_built() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + let mut built = eid("ids.example.com", vec![uid("a")]); + built.ext = Some(Cow::Owned( + RawValue::from_string(serde_json::json!({ "segments": ["s1"] }).to_string()) + .expect("should build raw"), + )); + + let (entries, dropped) = validate(vec![built], &policy(&provider, &claimed, &partners)); + + assert_eq!(dropped, DropCounts::default(), "should drop nothing"); + assert_eq!( + entries[0].1[0].fields().ext.as_ref().map(RawJson::to_value), + Some(serde_json::json!({ "segments": ["s1"] })), + "should keep the built ext" + ); + } } diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs index 151ebc853..006280533 100644 --- a/crates/trusted-server-core/src/ec/provider.rs +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -205,6 +205,9 @@ pub struct LookupInput<'a> { pub due_sources: &'a [SourceDomain], /// This provider's currently stored entries, borrowed from the row. pub current: OwnedEntries<'a>, + /// The request's cookies, for a vendor first-party cookie a lookup + /// sends as input. + pub cookies: RequestCookies<'a>, /// Always `None` in V1. pub hashed_email: Option<&'a HashedEmail>, } @@ -361,8 +364,8 @@ pub struct ProviderEid<'body> { pub matcher: Option>, /// `OpenRTB` `mm`. pub mm: Option, - /// EID-level `ext`. - pub ext: Option<&'body RawValue>, + /// EID-level `ext`, borrowed from the body or built by the provider. + pub ext: Option>, /// UIDs of the entry. pub uids: Vec>, } @@ -374,8 +377,8 @@ pub struct ProviderUid<'body> { pub id: Cow<'body, str>, /// Agent type; filled from partner config when missing. pub atype: Option, - /// UID-level `ext`. - pub ext: Option<&'body RawValue>, + /// UID-level `ext`, borrowed from the body or built by the provider. + pub ext: Option>, /// Expiry as the provider states it. pub expiry: ProviderExpiry, /// Private refresh state kept with the UID in the row and never served. diff --git a/crates/trusted-server-identity-providers/src/common.rs b/crates/trusted-server-identity-providers/src/common.rs index 2dc5138de..5861e5719 100644 --- a/crates/trusted-server-identity-providers/src/common.rs +++ b/crates/trusted-server-identity-providers/src/common.rs @@ -339,14 +339,14 @@ impl<'body> VendorEid<'body> { inserter: self.inserter, matcher: self.matcher, mm: self.mm, - ext: self.ext, + ext: self.ext.map(Cow::Borrowed), uids: self .uids .into_iter() .map(|uid| ProviderUid { id: uid.id, atype: uid.atype, - ext: uid.ext, + ext: uid.ext.map(Cow::Borrowed), expiry, }) .collect(), diff --git a/crates/trusted-server-identity-providers/src/id5.rs b/crates/trusted-server-identity-providers/src/id5.rs index 15b1ca86a..40050dd24 100644 --- a/crates/trusted-server-identity-providers/src/id5.rs +++ b/crates/trusted-server-identity-providers/src/id5.rs @@ -453,6 +453,7 @@ mod tests { publisher_domain: "www.example.com", due_sources: &[], current: OwnedEntries::new(None, &[]), + cookies: RequestCookies::default(), hashed_email: None, }; @@ -498,6 +499,7 @@ mod tests { publisher_domain: "www.example.com", due_sources: &[], current: OwnedEntries::new(None, &[]), + cookies: RequestCookies::default(), hashed_email: None, }; diff --git a/crates/trusted-server-identity-providers/src/liveintent.rs b/crates/trusted-server-identity-providers/src/liveintent.rs index ae17f15ba..a64f31a69 100644 --- a/crates/trusted-server-identity-providers/src/liveintent.rs +++ b/crates/trusted-server-identity-providers/src/liveintent.rs @@ -290,7 +290,9 @@ impl IdentityProvider for LiveIntentProvider { mod tests { use serde_json::json; use trusted_server_core::consent::types::ConsentContext; - use trusted_server_core::ec::provider::{ClientSignals, OwnedEntries, build_kind}; + use trusted_server_core::ec::provider::{ + ClientSignals, OwnedEntries, RequestCookies, build_kind, + }; use trusted_server_core::settings::{IdentityProviderConfig, ProviderSecrets}; use super::*; @@ -355,6 +357,7 @@ mod tests { publisher_domain: "www.example.com", due_sources: &due[..1], current: OwnedEntries::new(None, &[]), + cookies: RequestCookies::default(), hashed_email: None, }; @@ -495,6 +498,7 @@ mod tests { publisher_domain: "www.example.com", due_sources: &due, current: OwnedEntries::new(None, &[]), + cookies: RequestCookies::default(), hashed_email: None, }; diff --git a/crates/trusted-server-identity-providers/src/liveramp.rs b/crates/trusted-server-identity-providers/src/liveramp.rs index 384474002..168102573 100644 --- a/crates/trusted-server-identity-providers/src/liveramp.rs +++ b/crates/trusted-server-identity-providers/src/liveramp.rs @@ -318,6 +318,7 @@ mod tests { publisher_domain: "www.example.com", due_sources: &[], current: OwnedEntries::new(None, &[]), + cookies: RequestCookies::default(), hashed_email: None, }; From 7eb2026a2a84818594cbc7b50c6cc5f488a325c8 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 19:15:28 +0530 Subject: [PATCH 14/72] Send the LiveIntent first-party cookie and port its reply mapping Send _lc2_fpi as duid, resolve attributes from LiveIntent's fixed attribute table, keep reply segments as the nonId EID ext and mark the Trade Desk ID with matcher and rtiPartner, as the reference provider and Prebid do. Signed-off-by: dhruv8sh --- .../src/lib.rs | 4 +- .../src/liveintent.rs | 222 ++++++++++++++---- ...2026-11-02-identity-provider-liveintent.md | 25 +- trusted-server.example.toml | 31 ++- 4 files changed, 215 insertions(+), 67 deletions(-) diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index 37b2539f2..2a481bcd0 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -214,8 +214,8 @@ mod tests { ( "liveintent", Some("https://li.example.com/idex"), - json!(["li.example.com"]), - json!({ "attributes": { "nonId": "li.example.com" } }), + json!(["liveintent.com"]), + json!({ "resolve": ["nonId"] }), ), ("liveramp", None, json!(["liveramp.example.com"]), json!({})), ( diff --git a/crates/trusted-server-identity-providers/src/liveintent.rs b/crates/trusted-server-identity-providers/src/liveintent.rs index a64f31a69..bafbb9243 100644 --- a/crates/trusted-server-identity-providers/src/liveintent.rs +++ b/crates/trusted-server-identity-providers/src/liveintent.rs @@ -1,18 +1,23 @@ //! `LiveIntent` identity provider: resolves `LiveIntent`'s IDs server side. //! -//! Calls `LiveIntent`'s identity exchange with consent and network signals -//! and asks for the attributes the operator configured. Each attribute maps -//! to one configured `OpenRTB` source. The lookup is anonymous: no hashed -//! email and no first-party `LiveIntent` cookie are sent. +//! Calls `LiveIntent`'s identity exchange with the visitor's first-party +//! `LiveIntent` cookie (`_lc2_fpi` by default, sent as `duid`), consent and +//! network signals, and asks for the attributes the operator listed in +//! `resolve`. Each attribute maps to the `OpenRTB` source in +//! [`ATTRIBUTE_SOURCES`], which must be one of the provider's configured +//! sources. No hashed email is sent. //! //! As in Prebid's `LiveIntent` module, every attribute other than `nonId` is -//! a partner ID and its UID carries `ext.provider` naming `LiveIntent`. +//! a partner ID and its UID carries `ext.provider` naming `LiveIntent`; the +//! Trade Desk ID also carries `ext.rtiPartner` and its EID names +//! `LiveIntent` as `matcher`. `segments` in the reply become the `nonId` +//! EID's `ext`. //! //! This kind has no Lockr capability: Lockr AIM is not documented to //! deliver `LiveIntent` IDs, so a `lockr` settings table is rejected. use std::borrow::Cow; -use std::collections::{BTreeMap, HashMap}; +use std::collections::HashMap; use std::num::NonZeroUsize; use std::time::{Duration, SystemTime}; @@ -34,6 +39,38 @@ use url::Url; use crate::common::{bounded_expiry, is_routable_ip, join_section_ids}; +/// Each attribute `LiveIntent` resolves and the source it carries. +pub const ATTRIBUTE_SOURCES: &[(&str, &str)] = &[ + (NON_ID_ATTRIBUTE, "liveintent.com"), + ("bidswitch", "bidswitch.net"), + ("medianet", "media.net"), + ("magnite", "rubiconproject.com"), + ("index", "liveintent.indexexchange.com"), + ("openx", "openx.net"), + ("pubmatic", "pubmatic.com"), + ("sovrn", "liveintent.sovrn.com"), + ("sharethrough", "sharethrough.com"), + ("sonobi", "liveintent.sonobi.com"), + ("triplelift", "liveintent.triplelift.com"), + ("zetassp", "zeta-ssp.liveintent.com"), + ("vidazoo", "liveintent.vidazoo.com"), + ("nexxen", "liveintent.unrulymedia.com"), + ("uid2", "uidapi.com"), + (TDID_ATTRIBUTE, "adserver.org"), +]; + +/// The attribute holding the Trade Desk ID `LiveIntent` resolved. +const TDID_ATTRIBUTE: &str = "thetradedesk"; + +/// `matcher` of the Trade Desk ID EID: `LiveIntent` matched it. +const TDID_MATCHER: &str = "liveintent.com"; + +/// UID `ext` Prebid puts on the Trade Desk ID that `LiveIntent` resolved. +const TDID_UID_EXT: &str = r#"{"rtiPartner":"TDID","provider":"liveintent.com"}"#; + +/// `LiveIntent`'s default first-party ID cookie on the publisher's domain. +const FIRST_PARTY_COOKIE: &str = "_lc2_fpi"; + /// `atype` `LiveIntent` IDs carry: a person-based ID. const LIVEINTENT_ATYPE: i32 = 3; @@ -63,16 +100,28 @@ pub struct LiveIntentSettings { /// publisher has one. #[serde(default)] pub distributor_id: Option, - /// Map of `LiveIntent` attribute name to the source it is stored under. - /// - /// Every source must be one of the provider's configured `sources`. - pub attributes: BTreeMap, + /// The attributes to resolve, from [`ATTRIBUTE_SOURCES`]. Each one's + /// source must be one of the provider's configured `sources`. + #[serde(default = "default_resolve")] + pub resolve: Vec, + /// First-party `LiveIntent` cookies, in order; the first present one is + /// sent as `duid`. + #[serde(default = "default_cookie_names")] + pub cookie_names: Vec, } fn default_publisher_id() -> String { "any".to_owned() } +fn default_resolve() -> Vec { + vec![NON_ID_ATTRIBUTE.to_owned()] +} + +fn default_cookie_names() -> Vec { + vec![FIRST_PARTY_COOKIE.to_owned()] +} + /// A built `LiveIntent` provider. #[derive(Debug)] pub struct LiveIntentProvider { @@ -80,7 +129,9 @@ pub struct LiveIntentProvider { url: Url, distributor_id: Option, attributes: Box<[(String, SourceDomain)]>, + cookie_names: Box<[String]>, partner_uid_ext: &'static RawValue, + tdid_uid_ext: &'static RawValue, } /// Whether `id` has the `did-` form `LiveIntent` accepts; it answers any @@ -101,13 +152,20 @@ impl ProviderKind for LiveIntent { settings: LiveIntentSettings, ) -> Result> { let attributes = settings - .attributes + .resolve .into_iter() - .map(|(attribute, source)| { + .map(|attribute| { + let (_, source) = ATTRIBUTE_SOURCES + .iter() + .find(|(name, _)| *name == attribute) + .ok_or_else(|| { + Report::new(ProviderConfigError::Invalid) + .attach(format!("unknown LiveIntent attribute `{attribute}`")) + })?; let source = common .sources .iter() - .find(|known| known.as_str() == source) + .find(|known| known.as_str() == *source) .ok_or_else(|| { Report::new(ProviderConfigError::Invalid).attach(format!( "attribute `{attribute}` maps to `{source}`, which is not a configured source" @@ -131,6 +189,8 @@ impl ProviderKind for LiveIntent { } let partner_uid_ext = serde_json::from_str::<&'static RawValue>(PARTNER_UID_EXT) .change_context(ProviderConfigError::Invalid)?; + let tdid_uid_ext = serde_json::from_str::<&'static RawValue>(TDID_UID_EXT) + .change_context(ProviderConfigError::Invalid)?; let distributor = settings.distributor_id.as_deref().unwrap_or("prebid"); let mut url = common .endpoint @@ -158,13 +218,15 @@ impl ProviderKind for LiveIntent { shape: EidShape { max_eids, max_uids_per_eid: MAX_UIDS_PER_EID, - eid_level_fields: EidLevelFields::Ignored, + eid_level_fields: EidLevelFields::Preserved, }, }, url, distributor_id: settings.distributor_id, attributes, + cookie_names: settings.cookie_names.into_boxed_slice(), partner_uid_ext, + tdid_uid_ext, }) } } @@ -204,6 +266,13 @@ impl IdentityProvider for LiveIntentProvider { if let Some(distributor) = &self.distributor_id { query.append_pair("did", distributor); } + if let Some(first_party_id) = self + .cookie_names + .iter() + .find_map(|name| lookup.cookies.get(name)) + { + query.append_pair("duid", first_party_id); + } query.append_pair("gdpr", if consent.gdpr_applies { "1" } else { "0" }); if let Some(tc_string) = &consent.raw_tc_string { query.append_pair("gdpr_consent", tc_string); @@ -255,6 +324,9 @@ impl IdentityProvider for LiveIntentProvider { let reply: HashMap = serde_json::from_slice(response.body).change_context(ProviderError::InvalidResponse)?; let expiry = expiry(response.headers); + let segments = reply.get("segments").and_then(|segments| { + RawValue::from_string(format!(r#"{{"segments":{}}}"#, segments.get())).ok() + }); let eids: Vec> = self .attributes .iter() @@ -263,16 +335,25 @@ impl IdentityProvider for LiveIntentProvider { let id = Cow::<'body, str>::deserialize(*raw) .ok() .filter(|id| !id.is_empty())?; + let (matcher, eid_ext, uid_ext) = match attribute.as_str() { + NON_ID_ATTRIBUTE => (None, segments.clone().map(Cow::Owned), None), + TDID_ATTRIBUTE => ( + Some(Cow::Borrowed(TDID_MATCHER)), + None, + Some(self.tdid_uid_ext), + ), + _ => (None, None, Some(self.partner_uid_ext)), + }; Some(ProviderEid { source: Cow::Owned(source.as_str().to_owned()), inserter: None, - matcher: None, + matcher, mm: None, - ext: None, + ext: eid_ext, uids: vec![ProviderUid { id, atype: Some(LIVEINTENT_ATYPE), - ext: (attribute != NON_ID_ATTRIBUTE).then_some(self.partner_uid_ext), + ext: uid_ext.map(Cow::Borrowed), expiry, }], }) @@ -297,46 +378,85 @@ mod tests { use super::*; - fn config(attributes: &serde_json::Value) -> IdentityProviderConfig { + const SOURCES: [&str; 3] = ["liveintent.com", "adserver.org", "pubmatic.com"]; + + fn config(resolve: &serde_json::Value) -> IdentityProviderConfig { serde_json::from_value(json!({ "id": "liveintent", "kind": "liveintent", "endpoint": "https://li.example.com/idex", "allowed_domains": ["li.example.com"], - "sources": ["li.example.com", "li-partner.example.com"], - "settings": { "publisher_id": "12345", "attributes": attributes }, + "sources": SOURCES, + "settings": { "publisher_id": "12345", "resolve": resolve }, })) .expect("should parse provider config") } fn provider() -> Box { build_kind::( - &config(&json!({ "nonId": "li.example.com", "partner": "li-partner.example.com" })), + &config(&json!(["nonId", "thetradedesk", "pubmatic"])), &ProviderSecrets::default(), ) .expect("should build provider") } fn due() -> Vec { - ["li.example.com", "li-partner.example.com"] + SOURCES .into_iter() .map(|source| SourceDomain::try_from(source).expect("should parse source")) .collect() } #[test] - fn rejects_an_attribute_mapped_to_an_unconfigured_source() { - let error = build_kind::( - &config(&json!({ "nonId": "other.example.com" })), - &ProviderSecrets::default(), - ) - .err() - .expect("should reject the config"); + fn rejects_an_unknown_attribute_or_one_whose_source_is_not_configured() { + for resolve in [json!(["unknown"]), json!(["bidswitch"])] { + let error = build_kind::(&config(&resolve), &ProviderSecrets::default()) + .err() + .expect("should reject the config"); - assert!( - matches!(error.current_context(), ProviderConfigError::Invalid), - "should report invalid config" - ); + assert!( + matches!(error.current_context(), ProviderConfigError::Invalid), + "should report invalid config for {resolve}" + ); + } + } + + #[test] + fn sends_the_first_present_first_party_cookie_as_duid() { + let provider = provider(); + let consent = ConsentContext::default(); + let due = due(); + let lookup = |cookies| LookupInput { + ec_id: "ec-example", + consent: &consent, + client: ClientSignals { + ip: None, + user_agent: None, + }, + publisher_domain: "www.example.com", + due_sources: &due, + current: OwnedEntries::new(None, &[]), + cookies, + hashed_email: None, + }; + + for (header, expected) in [ + ("other=1; _lc2_fpi=fpi-example", vec!["fpi-example"]), + ("other=1; _lc2_fpi=", vec![]), + ("", vec![]), + ] { + let request = provider + .build_request(&lookup(RequestCookies::new(header))) + .expect("should build request"); + let duid: Vec = request + .url + .query_pairs() + .filter(|(key, _)| key == "duid") + .map(|(_, value)| value.into_owned()) + .collect(); + + assert_eq!(duid, expected, "should send duid for `{header}`"); + } } #[test] @@ -418,7 +538,9 @@ mod tests { // cannot hold. let body = json!({ "nonId": "li-example", - "partner": "partner-example", + "segments": ["s1", "s2"], + "thetradedesk": "tdid-example", + "pubmatic": "pubmatic-example", "unknown": "x", }) .to_string() @@ -436,8 +558,8 @@ mod tests { panic!("should find EIDs"); }; - assert_eq!(eids.len(), 2, "should return each configured attribute"); - assert_eq!(eids[0].source, "li.example.com", "should map nonId"); + assert_eq!(eids.len(), 3, "should return each configured attribute"); + assert_eq!(eids[0].source, "liveintent.com", "should map nonId"); assert_eq!(eids[0].uids[0].id, "li-example", "should keep the ID"); assert_eq!(eids[0].uids[0].atype, Some(3), "should set atype 3"); assert!( @@ -449,11 +571,29 @@ mod tests { "should keep nonId without ext" ); assert_eq!( - eids[1].source, "li-partner.example.com", - "should map partner" + eids[0].ext.as_deref().map(RawValue::get), + Some(r#"{"segments":["s1","s2"]}"#), + "should keep segments as the nonId EID ext" + ); + assert!( + eids[0].matcher.is_none(), + "should keep nonId without matcher" + ); + assert_eq!(eids[1].source, "adserver.org", "should map the TDID"); + assert_eq!( + eids[1].matcher.as_deref(), + Some("liveintent.com"), + "should name LiveIntent as the TDID matcher" + ); + assert_eq!( + eids[1].uids[0].ext.as_deref().map(RawValue::get), + Some(r#"{"rtiPartner":"TDID","provider":"liveintent.com"}"#), + "should mark the TDID as LiveIntent-provided" ); + assert!(eids[1].ext.is_none(), "should keep segments on nonId only"); + assert_eq!(eids[2].source, "pubmatic.com", "should map a partner ID"); assert_eq!( - eids[1].uids[0].ext.map(RawValue::get), + eids[2].uids[0].ext.as_deref().map(RawValue::get), Some(r#"{"provider":"liveintent.com"}"#), "should name LiveIntent as the provider of a partner ID" ); @@ -462,7 +602,7 @@ mod tests { #[test] fn rejects_a_distributor_id_liveintent_did_not_issue() { for distributor_id in ["prebid", "did-", "did-01/2"] { - let mut config = config(&json!({ "nonId": "li.example.com" })); + let mut config = config(&json!(["nonId"])); config .settings .insert("distributor_id".to_owned(), json!(distributor_id)); @@ -480,7 +620,7 @@ mod tests { #[test] fn calls_the_distributor_path_with_a_did_distributor_id() { - let mut config = config(&json!({ "nonId": "li.example.com" })); + let mut config = config(&json!(["nonId"])); config .settings .insert("distributor_id".to_owned(), json!("did-0123")); diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md b/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md index 6918d4e23..1d38c99c2 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md @@ -2,7 +2,7 @@ > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. -**Goal:** Resolve `LiveIntent`'s IDs at the edge as an anonymous `[[ec.identity_providers]]` entry with `kind = "liveintent"`, storing one EID per configured attribute. +**Goal:** Resolve `LiveIntent`'s IDs at the edge as a no-login `[[ec.identity_providers]]` entry with `kind = "liveintent"`, storing one EID per resolved attribute. **Architecture:** `crates/trusted-server-identity-providers/src/liveintent.rs` holds `LiveIntent` (a `ProviderKind`), `LiveIntentSettings` and `LiveIntentProvider` (an `IdentityProvider`), plus one `build` arm in `lib.rs`. The provider is pure: `build_request` builds one `GET` to the identity exchange, and `parse_response` maps returned attributes to sources. The coordinator from PR A applies consent, the source gate, validation, expiry and persistence. @@ -10,35 +10,36 @@ **Changes from the earlier LiveIntent plan (built on the old descriptor framework):** -- Anonymous only: no hashed email, and no first-party `LiveIntent` cookie (`duid`), because `LookupInput` carries no cookies. -- The attribute-to-source table is operator configuration (`settings.attributes`), not a hard-coded list; every mapped source must be one of the provider's `sources`. +- No login: no hashed email. The first-party `LiveIntent` cookie (`_lc2_fpi` by default) is sent as `duid`, read from `LookupInput::cookies`. +- The attribute-to-source table is fixed, as in the reference provider; `settings.resolve` picks attributes from it, and every picked attribute's source must be one of the provider's `sources`. - No page URL in `LookupInput`, so `pu` is the publisher origin. -- EID-level fields are ignored. As in Prebid, every attribute other than `nonId` is a partner ID whose UID carries a static `ext` of `{"provider":"liveintent.com"}`; no other `ext` (`segments`, `rtiPartner`) is synthesized. +- EID-level fields are preserved. As in Prebid, every attribute other than `nonId` is a partner ID whose UID carries `ext` `{"provider":"liveintent.com"}`; the Trade Desk ID's UID also carries `rtiPartner: "TDID"` and its EID `matcher = "liveintent.com"`. The reply's `segments` become the `nonId` EID's `ext` (`{"segments": ...}`), which needs the PR A change letting a provider return an owned EID `ext`. - The `Expires` header becomes `ProviderExpiry::At`; the provider never reads the clock. ## Settings -| Key | Default | Meaning | -| ---------------- | ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | -| `publisher_id` | `any` | Path segment after the distributor. | -| `distributor_id` | none | A `did-...` ID; sent as `did` and used as the path. LiveIntent rejects any other form, so build rejects it. The path uses `prebid` when unset. | -| `attributes` | required, ≥ 1 | `LiveIntent` attribute name → stored source. | +| Key | Default | Meaning | +| ---------------- | -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `publisher_id` | `any` | Path segment after the distributor. | +| `distributor_id` | none | A `did-...` ID; sent as `did` and used as the path. LiveIntent rejects any other form, so build rejects it. The path uses `prebid` when unset. | +| `resolve` | `["nonId"]` | Attributes to ask for, from the fixed table: `nonId` liveintent.com, `bidswitch`, `medianet`, `magnite`, `index`, `openx`, `pubmatic`, `sovrn`, `sharethrough`, `sonobi`, `triplelift`, `zetassp`, `vidazoo`, `nexxen`, `uid2`, `thetradedesk` adserver.org. | +| `cookie_names` | `["_lc2_fpi"]` | First-party cookies; the first present one is sent as `duid`. | No Lockr capability: Lockr AIM is not documented to deliver LiveIntent IDs, and the lookup uses LiveIntent's own signals, so a `[settings.lockr]` table is rejected. ## Request - `GET //` with `Origin: https://`. -- Query: `did`, `gdpr`, `gdpr_consent`, `us_privacy`, `gpp_s`, `gpp_as`, `pu`, `pip` or `pip6` (base64url of a routable client IP), and one `resolve` per attribute whose source is due. +- Query: `did`, `duid`, `gdpr`, `gdpr_consent`, `us_privacy`, `gpp_s`, `gpp_as`, `pu`, `pip` or `pip6` (base64url of a routable client IP), and one `resolve` per attribute whose source is due. ## Response - `204 No Content` is `LookupOutcome::NoMatch`. LiveIntent answers `204` to anonymous `gdpr=1` lookups and returns only `nonId` without its own third-party cookie; another non-2xx status or a non-object body is `ProviderError::InvalidResponse`. -- Each configured attribute with a non-empty string value becomes one EID with `atype = 3`, borrowed from the body. +- Each resolved attribute with a non-empty string value becomes one EID with `atype = 3`, borrowed from the body; `segments`, the TDID `matcher` and the UID `ext` values are added as above. ## Tasks - [ ] Add `liveintent.rs` and the `build` arm. -- [ ] Unit tests: unconfigured mapped source rejected, request shape with only due attributes, attributes mapped with the `Expires` expiry, no-content and empty IDs are no match, error status and malformed body rejected. +- [ ] Unit tests: unknown attribute and unconfigured source rejected, `duid` from the first present cookie, segments/matcher/ext mapping, request shape with only due attributes, attributes mapped with the `Expires` expiry, no-content and empty IDs are no match, error status and malformed body rejected. - [ ] Add a commented `kind = "liveintent"` example to `trusted-server.example.toml`. - [ ] Run `cargo test-fastly` and `cargo clippy-fastly`. diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 4c3e26bd6..5475f210a 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -177,28 +177,35 @@ pull_sync_concurrency = 3 # enabled = true # cookie_names = ["id5id"] # -# LiveIntent (kind "liveintent"): anonymous lookup, no secrets. Each attribute -# LiveIntent returns is stored under the source it maps to; every mapped -# source must be listed in `sources`. `nonId` is LiveIntent's own ID; the other -# attributes (Prebid's names, such as `bidswitch`, `triplelift`, `magnite`, -# `index`, `openx`, `pubmatic`, `sovrn` and `nexxen`) are partner IDs, sent -# with UID `ext.provider` naming LiveIntent. `distributor_id` is optional and -# must be a LiveIntent-issued `did-...` ID; without it the lookup uses the -# `prebid` path, as Prebid does. +# LiveIntent (kind "liveintent"): no hashed email, no secrets. Sends the +# visitor's first-party LiveIntent cookie (the first present one of +# `cookie_names`, default `_lc2_fpi`) as `duid`. `resolve` lists the attributes +# to ask for (default `nonId`); each one is stored under LiveIntent's fixed +# source for it, which must be listed in `sources`: nonId liveintent.com, +# bidswitch bidswitch.net, medianet media.net, magnite rubiconproject.com, +# index liveintent.indexexchange.com, openx openx.net, pubmatic pubmatic.com, +# sovrn liveintent.sovrn.com, sharethrough sharethrough.com, sonobi +# liveintent.sonobi.com, triplelift liveintent.triplelift.com, zetassp +# zeta-ssp.liveintent.com, vidazoo liveintent.vidazoo.com, nexxen +# liveintent.unrulymedia.com, uid2 uidapi.com, thetradedesk adserver.org. +# `nonId` is LiveIntent's own ID and carries the reply's `segments` as EID +# `ext`; the others are partner IDs, sent with UID `ext.provider` naming +# LiveIntent (the Trade Desk ID also with `rtiPartner` and EID `matcher`). +# `distributor_id` is optional and must be a LiveIntent-issued `did-...` ID; +# without it the lookup uses the `prebid` path, as Prebid does. # [[ec.identity_providers]] # id = "liveintent" # kind = "liveintent" # enabled = false # endpoint = "https://liveintent.example.com/idex" # allowed_domains = ["liveintent.example.com"] -# sources = ["liveintent.example.com", "ssp.example.com"] +# sources = ["liveintent.com", "rubiconproject.com"] # tcf_vendor_id = 9999 # [ec.identity_providers.settings] # publisher_id = "any" # distributor_id = "did-0123" -# [ec.identity_providers.settings.attributes] -# nonId = "liveintent.example.com" -# magnite = "ssp.example.com" +# resolve = ["nonId", "magnite"] +# cookie_names = ["_lc2_fpi"] # # LiveRamp (kind "liveramp"): no lookup and no endpoint. Reads the RampID # envelope from LiveRamp's own first-party `_lr_env` cookie and keeps it for From 9c8fdadde1f4cc83cfaff2a76fd2e352d018ef0a Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 19:17:14 +0530 Subject: [PATCH 15/72] Carry the ID5 linkType and keep ext from encoded ID5 cookies Signed-off-by: dhruv8sh --- .../src/id5.rs | 47 ++++++++++++++----- .../plans/2026-11-02-identity-provider-id5.md | 4 +- 2 files changed, 36 insertions(+), 15 deletions(-) diff --git a/crates/trusted-server-identity-providers/src/id5.rs b/crates/trusted-server-identity-providers/src/id5.rs index 40050dd24..9acefcac5 100644 --- a/crates/trusted-server-identity-providers/src/id5.rs +++ b/crates/trusted-server-identity-providers/src/id5.rs @@ -3,7 +3,8 @@ //! Posts the request's consent and network signals to ID5's fetch endpoint. //! It needs no login and sends no hashed email. ID5 returns complete EIDs //! under `ids`, or the legacy `universal_uid`, which is given to the first -//! configured source. +//! configured source with the reply's `ext.linkType` as UID `ext`, as +//! Prebid's ID5 module does. //! //! Parsing is lenient per entry: an entry of `ids`, `universal_uid` or //! `cache_control` with an unexpected shape is skipped rather than failing @@ -17,9 +18,9 @@ //! writes (`[settings.lockr]`, default key `id5id`). A cookie holding the ID5 fetch response JSON //! (plain or percent-encoded) is parsed exactly like a lookup response, and //! a bare ID string becomes one UID on the first source with agent type 1 -//! and the default TTL. The response `signature` is never stored. When a -//! percent-encoded JSON cookie is read, its `ext` objects are dropped, -//! since they cannot outlive the decoded copy. A cookie that yields no +//! and the default TTL. The response `signature` is never stored. A +//! percent-encoded JSON cookie is decoded and its entries, `ext` objects +//! included, are copied out of the decoded text. A cookie that yields no //! usable UID is ignored and the lookup runs as usual. //! //! The request field names for the client IP and user agent follow ID5's @@ -182,6 +183,14 @@ struct FetchReply<'body> { ids: Option<&'body RawValue>, #[serde(default, borrow)] cache_control: Option<&'body RawValue>, + #[serde(default, borrow)] + ext: Option<&'body RawValue>, +} + +#[derive(Deserialize)] +struct ReplyExt<'body> { + #[serde(rename = "linkType", default, borrow)] + link_type: Option<&'body RawValue>, } #[derive(Deserialize)] @@ -246,6 +255,13 @@ impl Id5Provider { .and_then(|raw| serde_json::from_str::>(raw.get()).ok()) .filter(|uid| is_usable_uid(uid)); if !has_primary && let Some(uid) = universal_uid { + let link_type = reply + .ext + .and_then(|raw| serde_json::from_str::>(raw.get()).ok()) + .and_then(|ext| ext.link_type) + .and_then(|link_type| { + RawValue::from_string(format!(r#"{{"linkType":{}}}"#, link_type.get())).ok() + }); eids.push(ProviderEid { source: Cow::Owned(primary.to_owned()), inserter: None, @@ -255,7 +271,7 @@ impl Id5Provider { uids: vec![ProviderUid { id: uid, atype: Some(1), - ext: None, + ext: link_type.map(Cow::Owned), expiry, }], }); @@ -273,7 +289,7 @@ impl Id5Provider { .parse_reply(value.as_bytes()) .ok()? .into_iter() - .map(owned_without_ext) + .map(owned) .collect(), value => { let id = value.trim(); @@ -369,22 +385,21 @@ impl IdentityProvider for Id5Provider { } } -/// Copies an EID parsed from a temporary string, dropping its `ext` -/// objects, which cannot be copied out of the borrowed JSON. -fn owned_without_ext<'any>(eid: ProviderEid<'_>) -> ProviderEid<'any> { +/// Copies an EID parsed from a temporary string so it outlives that string. +fn owned<'any>(eid: ProviderEid<'_>) -> ProviderEid<'any> { ProviderEid { source: Cow::Owned(eid.source.into_owned()), inserter: eid.inserter.map(|value| Cow::Owned(value.into_owned())), matcher: eid.matcher.map(|value| Cow::Owned(value.into_owned())), mm: eid.mm, - ext: None, + ext: eid.ext.map(|ext| Cow::Owned(ext.into_owned())), uids: eid .uids .into_iter() .map(|uid| ProviderUid { id: Cow::Owned(uid.id.into_owned()), atype: uid.atype, - ext: None, + ext: uid.ext.map(|ext| Cow::Owned(ext.into_owned())), expiry: uid.expiry, }) .collect(), @@ -545,6 +560,7 @@ mod tests { let headers = HeaderMap::new(); let body = serde_json::to_vec(&json!({ "universal_uid": "legacy-example", + "ext": { "linkType": 2 }, "ids": { "id5": { "eid": { "source": "id5.example.com", "uids": [{ "id": "0" }] } }, "broken": { "eid": "not an object" }, @@ -563,6 +579,11 @@ mod tests { ); assert_eq!(eids.len(), 1, "should drop the opted-out EID"); assert_eq!(eids[0].uids[0].id, "legacy-example", "should keep the UID"); + assert_eq!( + eids[0].uids[0].ext.as_deref().map(RawValue::get), + Some(r#"{"linkType":2}"#), + "should carry the reply linkType as UID ext" + ); assert!( matches!(eids[0].uids[0].expiry, ProviderExpiry::After(d) if d == crate::common::DEFAULT_TTL), "should use the default TTL without a cache max age" @@ -708,10 +729,10 @@ mod tests { "id5.example.com".to_owned(), "id5-example".to_owned(), Some(1), - false, + true, Duration::from_secs(7200) )], - "should decode the reply and drop ext objects" + "should decode the reply and keep ext objects" ); } diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-id5.md b/docs/superpowers/plans/2026-11-02-identity-provider-id5.md index 77d5f84c2..a9a1b5305 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-id5.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-id5.md @@ -27,7 +27,7 @@ - A non-2xx status or an unparseable body is `ProviderError::InvalidResponse`. - `ids..eid` entries whose source is configured become `ProviderEid`s; every field, including `ext`, is borrowed from the body. - `cache_control.max_age_sec` becomes `ProviderExpiry::After` for every UID. -- With no EID for the first source, a non-empty, non-`0` `universal_uid` becomes an EID with `atype = 1`. +- With no EID for the first source, a non-empty, non-`0` `universal_uid` becomes an EID with `atype = 1` and, when the reply has `ext.linkType`, UID `ext` `{"linkType": ...}`, as in the reference provider and Prebid. ## First-party cookie @@ -35,7 +35,7 @@ ID5's source is provider-owned, so the browser `ts-eids` copy is dropped. Prebid - Setting `cookie_names` (default `["id5id"]`), tried in order. - `id5` declares the Lockr capability (see `2026-11-02-identity-provider-lockr-capability.md`): with `[settings.lockr] enabled = true`, the Lockr cookies (default `["id5id"]`) are read after `cookie_names` and before the lookup, with the same parser. -- A value that is JSON after percent-decoding is parsed with the same code as a lookup response (lenient entries, opt-out `0` dropped, normalized sources, `cache_control.max_age_sec` bounded). `ext` objects are dropped when the value was percent-encoded, since they borrow from the decoded copy. +- A value that is JSON after percent-decoding is parsed with the same code as a lookup response (lenient entries, opt-out `0` dropped, normalized sources, `cache_control.max_age_sec` bounded). When the value was percent-encoded, entries, `ext` objects included, are copied out of the decoded text. - Any other value is a bare ID: one UID on the first source with `atype = 1` and the default TTL. - `signature` is never stored as a UID. A cookie that yields no usable UID is ignored, and the lookup runs as usual; a kept UID skips the lookup. From e4faab352614a75ab1c33b19883d14fb3876975d Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 19:17:50 +0530 Subject: [PATCH 16/72] Note the reference provider behaviours left unported Signed-off-by: dhruv8sh --- .../plans/2026-11-02-identity-provider-connectid.md | 4 ++++ .../plans/2026-11-02-identity-provider-liveramp.md | 4 ++++ .../plans/2026-11-02-identity-provider-lockr-capability.md | 4 ++++ docs/superpowers/plans/2026-11-02-identity-provider-uid2.md | 4 ++++ 4 files changed, 16 insertions(+) diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-connectid.md b/docs/superpowers/plans/2026-11-02-identity-provider-connectid.md index c10e2dbf1..7660c3eaa 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-connectid.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-connectid.md @@ -22,6 +22,10 @@ No Lockr capability: Lockr AIM is not documented to deliver ConnectID, so a `[se - One EID for the single configured source, one UID with `atype = 3`. - Expiry `ProviderExpiry::At(lastSynced + min(ttl, 14 days))`; a missing `ttl` uses the shared default TTL. The provider reads no clock: the coordinator's validation drops the ID once that time has passed. +## Not ported from the reference provider + +- The reference calls Yahoo's sync endpoint, which needs a hashed email on every call. The logged-in experience is out of scope, so nothing is ported; this provider reads the `connectId` cookie only. + ## Tasks - [ ] Add `connectid.rs` and register the kind in `is_registered_kind` and `build`. diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md b/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md index cbc581a31..8b220134a 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md @@ -24,6 +24,10 @@ `liveramp` declares the Lockr capability (see `2026-11-02-identity-provider-lockr-capability.md`). With `[settings.lockr] enabled = true`, the Lockr cookies (default `["_lr_env"]`) are read after `cookie_names` with the same envelope parser. `cookie_names` may be empty only when Lockr is enabled. +## Not ported from the reference provider + +- The reference refreshes a stored envelope server side (placement ID, approved countries, 15 or 30 days by region). It needs the visitor's region, which `LookupInput` does not carry, and its request and response shape are unconfirmed with LiveRamp, so this provider keeps reading the `_lr_env` cookie only. + ## Tasks - [ ] Add `cookie_capabilities`, `cookie_eid`, `is_usable_id` and `percent_decoded` to `common.rs`. diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-lockr-capability.md b/docs/superpowers/plans/2026-11-02-identity-provider-lockr-capability.md index ca4949cae..3fc31556c 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-lockr-capability.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-lockr-capability.md @@ -15,6 +15,10 @@ **Own-cookie settings:** every cookie-reading kind takes a `cookie_names` list (UID2's `token_cookie_name` and ConnectID's `cookie_name` are renamed). Lockr names move out of them into `[settings.lockr]`. +## Not ported from the reference provider + +- The reference captures proxied Lockr API responses: a `revoke-consent` call opts the visitor out of Lockr's sources, and token replies are mapped to sources with Lockr provenance (`inserter`, `mm`) on ID5. The framework has no response-capture hook or opt-out state, and Lockr's default ID5 key is the same `id5id` cookie Prebid writes, so a Lockr-read ID cannot be told apart for provenance. Neither is ported. + ## Tasks - [ ] Add `ProviderCommon::id` in core and set it in `build_kind`. diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md b/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md index b09c3f2bc..ae929d867 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md @@ -25,6 +25,10 @@ `uid2` declares the Lockr capability (see `2026-11-02-identity-provider-lockr-capability.md`). With `[settings.lockr] enabled = true`, the Lockr cookies (default `["__uid2_advertising_token"]`) are read last, after the identity cookie and `cookie_names`, with the raw-token parser. Lockr names already in `cookie_names` are not read twice. +## Not ported from the reference provider + +- The reference generates a first token from a hashed email and refreshes it with the refresh token it kept from that call. Generation needs login, and refreshing with the SDK cookie's refresh token would spend the browser's single-use refresh token without a way to write the new `__uid_2` cookie back, so neither is ported. + ## Tasks - [ ] Add `uid2.rs` and register the kind in `is_registered_kind` and `build`. From d956e90516d09d94cc4de68796955c9b5568cd16 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 20:11:47 +0530 Subject: [PATCH 17/72] Read the bare LiveRamp envelope Prebid stores in idl_env Prebid's identityLink ID module stores the envelope it fetches bare under its storage name, commonly idl_env, rather than the base64 JSON that ATS writes to _lr_env. Read idl_env after _lr_env by default and accept a base64-alphabet value that does not decode to JSON as a bare envelope. Signed-off-by: dhruv8sh --- .../src/liveramp.rs | 69 ++++++++++++++++--- .../2026-11-02-identity-provider-liveramp.md | 7 +- trusted-server.example.toml | 5 +- 3 files changed, 68 insertions(+), 13 deletions(-) diff --git a/crates/trusted-server-identity-providers/src/liveramp.rs b/crates/trusted-server-identity-providers/src/liveramp.rs index 168102573..13182e53d 100644 --- a/crates/trusted-server-identity-providers/src/liveramp.rs +++ b/crates/trusted-server-identity-providers/src/liveramp.rs @@ -3,6 +3,9 @@ //! //! `LiveRamp`'s ATS script stores the envelope in the `_lr_env` cookie as //! base64 (or base64url) of a JSON object whose `envelope` field holds it. +//! Prebid's `identityLink` ID module stores the envelope it receives bare, +//! under its configured storage name (commonly `idl_env`); a value that does +//! not decode to JSON is taken as such a bare envelope. //! The envelope is stored unchanged with agent type 3 for //! [`ENVELOPE_TTL`]. No server call is made: the refresh contract is not //! confirmed, so the browser keeps refreshing the cookie. @@ -11,8 +14,8 @@ //! AIM writes the envelope into (default key `_lr_env`) are read after the //! own cookies, with the same parser. //! -//! Parsing is lenient: an undecodable cookie, a body without a string -//! `envelope`, or a blank or opt-out envelope reads as no ID. +//! Parsing is lenient: a value outside the base64 alphabet, a JSON body +//! without a string `envelope`, or a blank or opt-out envelope reads as no ID. use std::borrow::Cow; use std::time::Duration; @@ -53,7 +56,7 @@ pub struct LiveRampSettings { impl LiveRampSettings { fn default_cookie_names() -> Vec { - vec!["_lr_env".to_owned()] + vec!["_lr_env".to_owned(), "idl_env".to_owned()] } } @@ -98,15 +101,33 @@ struct EnvelopeCookie { envelope: String, } -/// Decodes one `_lr_env` value into its envelope. +/// Decodes one cookie value into its envelope. +/// +/// A value whose base64 decodes to JSON must carry a usable `envelope` +/// (the ATS `_lr_env` format); any other base64-alphabet value is a bare +/// envelope (the Prebid `idl_env` format). fn envelope(value: &str) -> Option { let value = percent_decoded(value); let value = value.trim(); - let bytes = [STANDARD, STANDARD_NO_PAD, URL_SAFE, URL_SAFE_NO_PAD] + let is_base64_alphabet = value.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'+' | b'/' | b'=' | b'-' | b'_') + }); + if !is_base64_alphabet { + return None; + } + let json = [STANDARD, STANDARD_NO_PAD, URL_SAFE, URL_SAFE_NO_PAD] .iter() - .find_map(|engine| engine.decode(value).ok())?; - let cookie: EnvelopeCookie = serde_json::from_slice(&bytes).ok()?; - is_usable_id(&cookie.envelope).then_some(cookie.envelope) + .filter_map(|engine| engine.decode(value).ok()) + .find_map(|bytes| serde_json::from_slice::(&bytes).ok()); + let envelope = match json { + Some(body) => { + serde_json::from_value::(body) + .ok()? + .envelope + } + None => value.to_owned(), + }; + is_usable_id(&envelope).then_some(envelope) } impl IdentityProvider for LiveRampProvider { @@ -199,6 +220,36 @@ mod tests { } } + #[test] + fn reads_a_bare_prebid_idl_env_envelope() { + let provider = provider(&json!({})); + + let (id, _) = read(&*provider, "idl_env=AjfowMv4ZHZQJFM8TpiUnYEyA81Vdgg") + .expect("should read the bare envelope"); + + assert_eq!( + id, "AjfowMv4ZHZQJFM8TpiUnYEyA81Vdgg", + "should keep the bare envelope" + ); + } + + #[test] + fn prefers_lr_env_over_idl_env() { + let provider = provider(&json!({})); + let value = encoded(&json!({ "envelope": "env-ats" }), &STANDARD); + + let read = read( + &*provider, + &format!("idl_env=envPrebidExample; _lr_env={value}"), + ); + + assert_eq!( + read.map(|(id, _)| id).as_deref(), + Some("env-ats"), + "should prefer the ATS cookie" + ); + } + #[test] fn reads_a_configured_cookie_name() { let provider = provider(&json!({ "cookie_names": ["example_lockr_env"] })); @@ -263,6 +314,8 @@ mod tests { for header in [ String::new(), "_lr_env=not-base64!".to_owned(), + "idl_env=0".to_owned(), + "idl_env=".to_owned(), format!("_lr_env={}", encoded(&json!({ "other": "x" }), &STANDARD)), format!("_lr_env={}", encoded(&json!({ "envelope": 5 }), &STANDARD)), format!("_lr_env={}", encoded(&json!({ "envelope": "" }), &STANDARD)), diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md b/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md index 8b220134a..b17f97a65 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md @@ -10,9 +10,10 @@ ## Cookie -- Default cookie `_lr_env`; `cookie_names` overrides the list, read in order. -- The value is base64 or base64url (padded or not, optionally percent-encoded) of a JSON object. Only its string `envelope` field is used, unchanged. -- A blank or opt-out envelope (`0`, all zeros) and any undecodable value read as no ID. +- Default cookies `_lr_env` then `idl_env`; `cookie_names` overrides the list, read in order. +- `_lr_env` (written by ATS) is base64 or base64url (padded or not, optionally percent-encoded) of a JSON object. Only its string `envelope` field is used, unchanged. +- `idl_env` is where Prebid's `identityLink` ID module stores the envelope it fetched, bare. A value in the base64 alphabet that does not decode to JSON is taken as such a bare envelope, unchanged. +- A blank or opt-out envelope (`0`, all zeros), a JSON body without a string `envelope`, and any value outside the base64 alphabet read as no ID. ## Stored ID diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 5475f210a..e2ff8c459 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -208,7 +208,8 @@ pull_sync_concurrency = 3 # cookie_names = ["_lc2_fpi"] # # LiveRamp (kind "liveramp"): no lookup and no endpoint. Reads the RampID -# envelope from LiveRamp's own first-party `_lr_env` cookie and keeps it for +# envelope from LiveRamp's own first-party `_lr_env` cookie (ATS) or the bare +# envelope Prebid's identityLink module stores in `idl_env`, and keeps it for # 15 days. Takes exactly one source; use the one LiveRamp gives you. Supports # the Lockr capability. # [[ec.identity_providers]] @@ -218,7 +219,7 @@ pull_sync_concurrency = 3 # sources = ["liveramp.example.com"] # tcf_vendor_id = 9999 # [ec.identity_providers.settings] -# cookie_names = ["_lr_env"] +# cookie_names = ["_lr_env", "idl_env"] # [ec.identity_providers.settings.lockr] # enabled = true # cookie_names = ["_lr_env"] From 82e662d5cea7f5b8c9d223e5827bb6ab8a3dec70 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 22:00:02 +0530 Subject: [PATCH 18/72] Add a SharedID identity provider reading the Prebid first-party cookie Reads the publisher common ID from _sharedid, falling back to _pubcid, as a device-based pubcid.org EID, and honours the _pubcid_optout cookie. Signed-off-by: dhruv8sh --- .../src/common.rs | 18 ++ .../src/lib.rs | 10 ++ .../src/sharedid.rs | 163 ++++++++++++++++++ .../2026-11-02-identity-provider-sharedid.md | 30 ++++ trusted-server.example.toml | 12 ++ 5 files changed, 233 insertions(+) create mode 100644 crates/trusted-server-identity-providers/src/sharedid.rs create mode 100644 docs/superpowers/plans/2026-11-02-identity-provider-sharedid.md diff --git a/crates/trusted-server-identity-providers/src/common.rs b/crates/trusted-server-identity-providers/src/common.rs index 5861e5719..c034c0f49 100644 --- a/crates/trusted-server-identity-providers/src/common.rs +++ b/crates/trusted-server-identity-providers/src/common.rs @@ -179,6 +179,24 @@ pub(crate) fn cookie_eid<'cookie>( } } +/// Agent type of the device-based IDs read from browser-ID cookies. +pub(crate) const DEVICE_ATYPE: i32 = 1; + +/// One EID for `source` holding the single device-based UID `id` with +/// agent type 1. +#[must_use] +pub(crate) fn device_eid<'cookie>( + source: &SourceDomain, + id: Cow<'cookie, str>, + expiry: ProviderExpiry, +) -> ProviderEid<'cookie> { + let mut eid = cookie_eid(source, id, expiry); + for uid in &mut eid.uids { + uid.atype = Some(DEVICE_ATYPE); + } + eid +} + /// Percent-decodes a cookie value, keeping it unchanged when it is not /// valid percent-encoded UTF-8. #[must_use] diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index 2a481bcd0..0b4ec99f1 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -16,6 +16,7 @@ pub mod connectid; pub mod id5; pub mod liveintent; pub mod liveramp; +pub mod sharedid; pub mod uid2; use std::sync::Arc; @@ -43,6 +44,7 @@ pub fn is_registered_kind(kind: &str) -> bool { | liveramp::LiveRamp::KIND | connectid::ConnectId::KIND | uid2::Uid2::KIND + | sharedid::SharedId::KIND ) } @@ -86,6 +88,7 @@ pub fn build( liveramp::LiveRamp::KIND => build_kind::(config, secrets), connectid::ConnectId::KIND => build_kind::(config, secrets), uid2::Uid2::KIND => build_kind::(config, secrets), + sharedid::SharedId::KIND => build_kind::(config, secrets), kind => Err(Report::new(ProviderConfigError::UnknownKind) .attach(format!("kind `{kind}` is not registered"))), } @@ -225,6 +228,13 @@ mod tests { json!({}), ), ("uid2", None, json!(["uid2.example.com"]), json!({})), + ("sharedid", None, json!(["sharedid.example.com"]), json!({})), + ( + "unifiedid", + None, + json!(["unifiedid.example.com"]), + json!({}), + ), ] { let mut settings = settings; settings["lockr"] = json!({ "enabled": true }); diff --git a/crates/trusted-server-identity-providers/src/sharedid.rs b/crates/trusted-server-identity-providers/src/sharedid.rs new file mode 100644 index 000000000..f53e1b97d --- /dev/null +++ b/crates/trusted-server-identity-providers/src/sharedid.rs @@ -0,0 +1,163 @@ +//! Prebid `SharedID` identity provider: reads the publisher common ID from +//! the first-party cookie Prebid's `sharedIdSystem` writes. +//! +//! Prebid stores the ID as a bare UUID in `_sharedid` (the publisher's +//! configured name) or the legacy `_pubcid`. It is sent as a device-based +//! (agent type 1) EID. The opt-out cookie `_pubcid_optout` suppresses it. +//! No server call is made. + +use std::borrow::Cow; + +use error_stack::Report; +use serde::Deserialize; +use trusted_server_core::ec::eid_types::SourceDomain; +use trusted_server_core::ec::provider::{ + IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, + ProviderConfigError, ProviderEid, ProviderError, ProviderKind, ProviderRequest, + ProviderResponse, RequestCookies, +}; + +use crate::common::{CookieReader, bounded_expiry, cookie_capabilities, device_eid, is_usable_id}; + +/// Cookie Prebid sets when the user opts out of `SharedID`. +const OPT_OUT_COOKIE: &str = "_pubcid_optout"; + +/// The Prebid `SharedID` provider kind. +pub struct SharedId; + +/// Settings for a `sharedid` provider. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SharedIdSettings { + /// Cookies holding the `SharedID`, read in order. + #[serde(default = "SharedIdSettings::default_cookie_names")] + pub cookie_names: Vec, +} + +impl SharedIdSettings { + fn default_cookie_names() -> Vec { + vec!["_sharedid".to_owned(), "_pubcid".to_owned()] + } +} + +/// A built Prebid `SharedID` provider. +#[derive(Debug)] +pub struct SharedIdProvider { + capabilities: ProviderCapabilities, + source: SourceDomain, + cookies: CookieReader, +} + +impl ProviderKind for SharedId { + const KIND: &'static str = "sharedid"; + type Settings = SharedIdSettings; + type Provider = SharedIdProvider; + + fn build( + common: &ProviderCommon<'_>, + settings: SharedIdSettings, + ) -> Result> { + if settings.cookie_names.is_empty() { + return Err(Report::new(ProviderConfigError::Invalid) + .attach("a sharedid provider needs at least one cookie name")); + } + let (capabilities, source) = cookie_capabilities(common, Self::KIND)?; + Ok(SharedIdProvider { + capabilities, + source, + cookies: CookieReader::own(common, settings.cookie_names), + }) + } +} + +impl IdentityProvider for SharedIdProvider { + fn capabilities(&self) -> &ProviderCapabilities { + &self.capabilities + } + + fn build_request( + &self, + _lookup: &LookupInput<'_>, + ) -> Result> { + Err(Report::new(ProviderError::Request).attach("sharedid makes no lookup")) + } + + fn parse_response<'body>( + &self, + _response: ProviderResponse<'body>, + ) -> Result, Report> { + Err(Report::new(ProviderError::InvalidResponse).attach("sharedid makes no lookup")) + } + + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> Option>> { + if cookies.get(OPT_OUT_COOKIE).is_some() { + return None; + } + let id = self + .cookies + .read(cookies, |value| is_usable_id(value).then_some(value.trim()))?; + Some(vec![device_eid( + &self.source, + Cow::Borrowed(id), + bounded_expiry(None), + )]) + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + use trusted_server_core::ec::provider::build_kind; + use trusted_server_core::settings::{IdentityProviderConfig, ProviderSecrets}; + + use super::*; + + fn provider() -> Box { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "sharedid", + "kind": "sharedid", + "sources": ["sharedid.example.com"], + })) + .expect("should parse provider config"); + build_kind::(&config, &ProviderSecrets::default()).expect("should build provider") + } + + fn read(header: &str) -> Option { + let eids = provider().read_request_cookies(RequestCookies::new(header))?; + assert_eq!( + eids[0].source, "sharedid.example.com", + "should use the source" + ); + let uid = &eids[0].uids[0]; + assert_eq!(uid.atype, Some(1), "should be device-based"); + Some(uid.id.to_string()) + } + + #[test] + fn reads_sharedid_before_pubcid() { + assert_eq!( + read("_pubcid=pubcid-example; _sharedid=sharedid-example").as_deref(), + Some("sharedid-example"), + "should prefer _sharedid" + ); + assert_eq!( + read("_pubcid=pubcid-example").as_deref(), + Some("pubcid-example"), + "should fall back to _pubcid" + ); + } + + #[test] + fn honours_opt_out_and_ignores_blank_ids() { + assert!( + read("_sharedid=sharedid-example; _pubcid_optout=1").is_none(), + "should honour the opt-out cookie" + ); + for header in ["_sharedid=", "_sharedid=0", "other=1"] { + assert!(read(header).is_none(), "should ignore {header}"); + } + } +} diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-sharedid.md b/docs/superpowers/plans/2026-11-02-identity-provider-sharedid.md new file mode 100644 index 000000000..8a4e3e2c8 --- /dev/null +++ b/docs/superpowers/plans/2026-11-02-identity-provider-sharedid.md @@ -0,0 +1,30 @@ +# Prebid SharedID Identity Provider Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Store the Prebid SharedID (publisher common ID) from its first-party cookie, as an `[[ec.identity_providers]]` entry with `kind = "sharedid"`. No server call. + +**Architecture:** `crates/trusted-server-identity-providers/src/sharedid.rs` holds `SharedId` (a `ProviderKind`), `SharedIdSettings` and `SharedIdProvider`. The provider implements only `IdentityProvider::read_request_cookies`; `build_request` always fails, so no lookup is sent. `common::device_eid` builds the agent type 1 EID. + +**Spec:** `docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md`, section 7. + +## Cookie + +- Prebid's `sharedIdSystem` writes a bare UUID to the publisher-configured cookie, usually `_sharedid`; older pages use `_pubcid`. +- Default `cookie_names = ["_sharedid", "_pubcid"]`, read in order. +- The `_pubcid_optout` cookie suppresses the ID, as in Prebid. +- A blank or opt-out (all-zero) value reads as no ID. + +No Lockr capability. + +## Stored ID + +- One EID for the single configured source, one UID with `atype = 1`, as Prebid sends `pubcid.org`. +- Default bounded expiry; each request carrying the cookie refreshes it. + +## Tasks + +- [ ] Add `sharedid.rs` and `common::device_eid`; register the kind in `is_registered_kind` and `build`. +- [ ] Unit tests: `_sharedid` before `_pubcid`, opt-out cookie honoured, blank and opt-out IDs ignored. +- [ ] Add a commented `kind = "sharedid"` example to `trusted-server.example.toml`. +- [ ] Run `cargo test-fastly` and `cargo clippy-fastly`. diff --git a/trusted-server.example.toml b/trusted-server.example.toml index e2ff8c459..47fc32d76 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -253,6 +253,18 @@ pull_sync_concurrency = 3 # [ec.identity_providers.settings.lockr] # enabled = true # cookie_names = ["__uid2_advertising_token"] +# +# Prebid SharedID (kind "sharedid"): no lookup and no endpoint. Reads the +# publisher common ID from Prebid's first-party `_sharedid` cookie, falling +# back to the legacy `_pubcid`, as a device-based (atype 1) ID. The +# `_pubcid_optout` cookie suppresses it. Takes exactly one source. +# [[ec.identity_providers]] +# id = "sharedid" +# kind = "sharedid" +# enabled = false +# sources = ["sharedid.example.com"] +# [ec.identity_providers.settings] +# cookie_names = ["_sharedid", "_pubcid"] # ============================================================================= From a7ba91058e8f063ba3a22f018f3f50cd5223d74e Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 22:00:02 +0530 Subject: [PATCH 19/72] Add a Criteo identity provider reading the cto_bidid cookie Stores Criteo's bid ID verbatim as the device-based EID Prebid sends, with no server call. Signed-off-by: dhruv8sh --- .../src/criteo.rs | 150 ++++++++++++++++++ .../src/lib.rs | 4 + .../2026-11-02-identity-provider-criteo.md | 33 ++++ trusted-server.example.toml | 11 ++ 4 files changed, 198 insertions(+) create mode 100644 crates/trusted-server-identity-providers/src/criteo.rs create mode 100644 docs/superpowers/plans/2026-11-02-identity-provider-criteo.md diff --git a/crates/trusted-server-identity-providers/src/criteo.rs b/crates/trusted-server-identity-providers/src/criteo.rs new file mode 100644 index 000000000..21ff79cf6 --- /dev/null +++ b/crates/trusted-server-identity-providers/src/criteo.rs @@ -0,0 +1,150 @@ +//! Criteo identity provider: reads the Criteo ID from the first-party +//! cookie Prebid's `criteoIdSystem` writes. +//! +//! Prebid stores Criteo's bid ID in `cto_bidid` and sends that value +//! unchanged (still percent-encoded) as the `criteo.com` EID with agent +//! type 1, so it is stored verbatim. No server call is made. + +use std::borrow::Cow; + +use error_stack::Report; +use serde::Deserialize; +use trusted_server_core::ec::eid_types::SourceDomain; +use trusted_server_core::ec::provider::{ + IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, + ProviderConfigError, ProviderEid, ProviderError, ProviderKind, ProviderRequest, + ProviderResponse, RequestCookies, +}; + +use crate::common::{CookieReader, bounded_expiry, cookie_capabilities, device_eid, is_usable_id}; + +/// The Criteo provider kind. +pub struct Criteo; + +/// Settings for a `criteo` provider. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CriteoSettings { + /// Cookies holding the Criteo bid ID, read in order. + #[serde(default = "CriteoSettings::default_cookie_names")] + pub cookie_names: Vec, +} + +impl CriteoSettings { + fn default_cookie_names() -> Vec { + vec!["cto_bidid".to_owned()] + } +} + +/// A built Criteo provider. +#[derive(Debug)] +pub struct CriteoProvider { + capabilities: ProviderCapabilities, + source: SourceDomain, + cookies: CookieReader, +} + +impl ProviderKind for Criteo { + const KIND: &'static str = "criteo"; + type Settings = CriteoSettings; + type Provider = CriteoProvider; + + fn build( + common: &ProviderCommon<'_>, + settings: CriteoSettings, + ) -> Result> { + if settings.cookie_names.is_empty() { + return Err(Report::new(ProviderConfigError::Invalid) + .attach("a criteo provider needs at least one cookie name")); + } + let (capabilities, source) = cookie_capabilities(common, Self::KIND)?; + Ok(CriteoProvider { + capabilities, + source, + cookies: CookieReader::own(common, settings.cookie_names), + }) + } +} + +impl IdentityProvider for CriteoProvider { + fn capabilities(&self) -> &ProviderCapabilities { + &self.capabilities + } + + fn build_request( + &self, + _lookup: &LookupInput<'_>, + ) -> Result> { + Err(Report::new(ProviderError::Request).attach("criteo makes no lookup")) + } + + fn parse_response<'body>( + &self, + _response: ProviderResponse<'body>, + ) -> Result, Report> { + Err(Report::new(ProviderError::InvalidResponse).attach("criteo makes no lookup")) + } + + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> Option>> { + let id = self + .cookies + .read(cookies, |value| is_usable_id(value).then_some(value.trim()))?; + Some(vec![device_eid( + &self.source, + Cow::Borrowed(id), + bounded_expiry(None), + )]) + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + use trusted_server_core::ec::provider::build_kind; + use trusted_server_core::settings::{IdentityProviderConfig, ProviderSecrets}; + + use super::*; + + fn provider() -> Box { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "criteo", + "kind": "criteo", + "sources": ["criteo.example.com"], + })) + .expect("should parse provider config"); + build_kind::(&config, &ProviderSecrets::default()).expect("should build provider") + } + + #[test] + fn stores_the_bid_id_verbatim() { + let eids = provider() + .read_request_cookies(RequestCookies::new("cto_bidid=bidid-example%3D%3D")) + .expect("should read the bid ID"); + + assert_eq!( + eids[0].source, "criteo.example.com", + "should use the source" + ); + let uid = &eids[0].uids[0]; + assert_eq!( + uid.id, "bidid-example%3D%3D", + "should keep the value encoded" + ); + assert_eq!(uid.atype, Some(1), "should be device-based"); + } + + #[test] + fn ignores_missing_and_blank_cookies() { + for header in ["cto_bidid=", "cto_bidid=0", "cto_bundle=bundle-example"] { + assert!( + provider() + .read_request_cookies(RequestCookies::new(header)) + .is_none(), + "should ignore {header}" + ); + } + } +} diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index 0b4ec99f1..40066ff97 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -13,6 +13,7 @@ mod common; pub use common::LockrSettings; pub mod connectid; +pub mod criteo; pub mod id5; pub mod liveintent; pub mod liveramp; @@ -45,6 +46,7 @@ pub fn is_registered_kind(kind: &str) -> bool { | connectid::ConnectId::KIND | uid2::Uid2::KIND | sharedid::SharedId::KIND + | criteo::Criteo::KIND ) } @@ -89,6 +91,7 @@ pub fn build( connectid::ConnectId::KIND => build_kind::(config, secrets), uid2::Uid2::KIND => build_kind::(config, secrets), sharedid::SharedId::KIND => build_kind::(config, secrets), + criteo::Criteo::KIND => build_kind::(config, secrets), kind => Err(Report::new(ProviderConfigError::UnknownKind) .attach(format!("kind `{kind}` is not registered"))), } @@ -229,6 +232,7 @@ mod tests { ), ("uid2", None, json!(["uid2.example.com"]), json!({})), ("sharedid", None, json!(["sharedid.example.com"]), json!({})), + ("criteo", None, json!(["criteo.example.com"]), json!({})), ( "unifiedid", None, diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-criteo.md b/docs/superpowers/plans/2026-11-02-identity-provider-criteo.md new file mode 100644 index 000000000..c33edb060 --- /dev/null +++ b/docs/superpowers/plans/2026-11-02-identity-provider-criteo.md @@ -0,0 +1,33 @@ +# Criteo Identity Provider Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Store the Criteo ID from Criteo's first-party cookie, as an `[[ec.identity_providers]]` entry with `kind = "criteo"`. No server call. + +**Architecture:** `crates/trusted-server-identity-providers/src/criteo.rs` holds `Criteo` (a `ProviderKind`), `CriteoSettings` and `CriteoProvider`. The provider implements only `IdentityProvider::read_request_cookies`; `build_request` always fails. + +**Spec:** `docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md`, section 7. + +## Cookie + +- Prebid's `criteoIdSystem` keeps Criteo's bid ID in the `cto_bidid` cookie (and local storage) and sends that value unchanged, still percent-encoded, as the `criteo.com` EID. `cto_bundle` is Criteo's own sync state and is not an ID. +- Default `cookie_names = ["cto_bidid"]`, read in order. The value is stored verbatim, not decoded. +- A blank or opt-out value reads as no ID. Criteo does not set `cto_bidid` on every page view, so the ID arrives on a later request. + +No Lockr capability. + +## Stored ID + +- One EID for the single configured source, one UID with `atype = 1`. +- Default bounded expiry; each request carrying the cookie refreshes it. + +## Not ported + +- Criteo's `gum.criteo.com` sync call needs Criteo's third-party cookie, so it is not made; the first-party cookie is enough. + +## Tasks + +- [ ] Add `criteo.rs`; register the kind in `is_registered_kind` and `build`. +- [ ] Unit tests: value stored verbatim, missing and blank cookies ignored. +- [ ] Add a commented `kind = "criteo"` example to `trusted-server.example.toml`. +- [ ] Run `cargo test-fastly` and `cargo clippy-fastly`. diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 47fc32d76..02c5c7a02 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -265,6 +265,17 @@ pull_sync_concurrency = 3 # sources = ["sharedid.example.com"] # [ec.identity_providers.settings] # cookie_names = ["_sharedid", "_pubcid"] +# +# Criteo (kind "criteo"): no lookup and no endpoint. Reads Criteo's bid ID +# from the first-party `cto_bidid` cookie Prebid's Criteo module keeps and +# stores it verbatim as a device-based (atype 1) ID. Takes exactly one source. +# [[ec.identity_providers]] +# id = "criteo" +# kind = "criteo" +# enabled = false +# sources = ["criteo.example.com"] +# [ec.identity_providers.settings] +# cookie_names = ["cto_bidid"] # ============================================================================= From 666096e48069cc1f13cd5062df21872651f62fc7 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 22:00:02 +0530 Subject: [PATCH 20/72] Add a Unified ID identity provider reading the pbjs-unifiedid cookie Stores The Trade Desk ID with the rtiPartner ext, inserter, matcher and mm that Prebid sets. Signed-off-by: dhruv8sh --- .../src/lib.rs | 3 + .../src/unifiedid.rs | 219 ++++++++++++++++++ .../2026-11-02-identity-provider-unifiedid.md | 30 +++ trusted-server.example.toml | 13 ++ 4 files changed, 265 insertions(+) create mode 100644 crates/trusted-server-identity-providers/src/unifiedid.rs create mode 100644 docs/superpowers/plans/2026-11-02-identity-provider-unifiedid.md diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index 40066ff97..0b5caaae3 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -19,6 +19,7 @@ pub mod liveintent; pub mod liveramp; pub mod sharedid; pub mod uid2; +pub mod unifiedid; use std::sync::Arc; use std::time::Duration; @@ -47,6 +48,7 @@ pub fn is_registered_kind(kind: &str) -> bool { | uid2::Uid2::KIND | sharedid::SharedId::KIND | criteo::Criteo::KIND + | unifiedid::UnifiedId::KIND ) } @@ -92,6 +94,7 @@ pub fn build( uid2::Uid2::KIND => build_kind::(config, secrets), sharedid::SharedId::KIND => build_kind::(config, secrets), criteo::Criteo::KIND => build_kind::(config, secrets), + unifiedid::UnifiedId::KIND => build_kind::(config, secrets), kind => Err(Report::new(ProviderConfigError::UnknownKind) .attach(format!("kind `{kind}` is not registered"))), } diff --git a/crates/trusted-server-identity-providers/src/unifiedid.rs b/crates/trusted-server-identity-providers/src/unifiedid.rs new file mode 100644 index 000000000..56409eb8c --- /dev/null +++ b/crates/trusted-server-identity-providers/src/unifiedid.rs @@ -0,0 +1,219 @@ +//! Unified ID identity provider: reads The Trade Desk ID (TDID) from the +//! first-party cookie Prebid's `unifiedIdSystem` writes. +//! +//! Prebid stores a percent-encoded JSON object such as `{"TDID":"…"}` in +//! `pbjs-unifiedid` and sends the TDID as the `adserver.org` EID with agent +//! type 1, UID `ext` `{"rtiPartner":"TDID"}`, and `inserter`, `matcher` set +//! to `adserver.org` with `mm` 4. No server call is made. + +use std::borrow::Cow; + +use error_stack::{Report, ResultExt as _}; +use serde::Deserialize; +use serde_json::value::RawValue; +use trusted_server_core::ec::eid_types::SourceDomain; +use trusted_server_core::ec::provider::{ + EidLevelFields, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, + ProviderCommon, ProviderConfigError, ProviderEid, ProviderError, ProviderKind, ProviderRequest, + ProviderResponse, RequestCookies, +}; + +use crate::common::{ + CookieReader, bounded_expiry, cookie_capabilities, device_eid, is_usable_id, percent_decoded, +}; + +/// `inserter` and `matcher` Prebid sets on the TDID EID. +const TDID_INSERTER: &str = "adserver.org"; + +/// `mm` (match method) Prebid sets on the TDID EID. +const TDID_MATCH_METHOD: i32 = 4; + +/// UID `ext` Prebid puts on the TDID. +const TDID_UID_EXT: &str = r#"{"rtiPartner":"TDID"}"#; + +/// The Unified ID provider kind. +pub struct UnifiedId; + +/// Settings for a `unifiedid` provider. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct UnifiedIdSettings { + /// Cookies holding the Unified ID object, read in order. + #[serde(default = "UnifiedIdSettings::default_cookie_names")] + pub cookie_names: Vec, +} + +impl UnifiedIdSettings { + fn default_cookie_names() -> Vec { + vec!["pbjs-unifiedid".to_owned()] + } +} + +/// A built Unified ID provider. +#[derive(Debug)] +pub struct UnifiedIdProvider { + capabilities: ProviderCapabilities, + source: SourceDomain, + cookies: CookieReader, + uid_ext: &'static RawValue, +} + +impl ProviderKind for UnifiedId { + const KIND: &'static str = "unifiedid"; + type Settings = UnifiedIdSettings; + type Provider = UnifiedIdProvider; + + fn build( + common: &ProviderCommon<'_>, + settings: UnifiedIdSettings, + ) -> Result> { + if settings.cookie_names.is_empty() { + return Err(Report::new(ProviderConfigError::Invalid) + .attach("a unifiedid provider needs at least one cookie name")); + } + let (mut capabilities, source) = cookie_capabilities(common, Self::KIND)?; + capabilities.shape.eid_level_fields = EidLevelFields::Preserved; + let uid_ext = serde_json::from_str::<&'static RawValue>(TDID_UID_EXT) + .change_context(ProviderConfigError::Invalid)?; + Ok(UnifiedIdProvider { + capabilities, + source, + cookies: CookieReader::own(common, settings.cookie_names), + uid_ext, + }) + } +} + +/// The field of the Unified ID cookie this provider reads. +#[derive(Deserialize)] +struct UnifiedIdCookie { + #[serde(rename = "TDID")] + tdid: String, +} + +/// Decodes one cookie value into the TDID. +fn tdid(value: &str) -> Option { + let cookie: UnifiedIdCookie = serde_json::from_str(&percent_decoded(value)).ok()?; + is_usable_id(&cookie.tdid).then_some(cookie.tdid) +} + +impl IdentityProvider for UnifiedIdProvider { + fn capabilities(&self) -> &ProviderCapabilities { + &self.capabilities + } + + fn build_request( + &self, + _lookup: &LookupInput<'_>, + ) -> Result> { + Err(Report::new(ProviderError::Request).attach("unifiedid makes no lookup")) + } + + fn parse_response<'body>( + &self, + _response: ProviderResponse<'body>, + ) -> Result, Report> { + Err(Report::new(ProviderError::InvalidResponse).attach("unifiedid makes no lookup")) + } + + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> Option>> { + let id = self.cookies.read(cookies, tdid)?; + let mut eid = device_eid(&self.source, Cow::Owned(id), bounded_expiry(None)); + eid.inserter = Some(Cow::Borrowed(TDID_INSERTER)); + eid.matcher = Some(Cow::Borrowed(TDID_INSERTER)); + eid.mm = Some(TDID_MATCH_METHOD); + for uid in &mut eid.uids { + uid.ext = Some(Cow::Borrowed(self.uid_ext)); + } + Some(vec![eid]) + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + use trusted_server_core::ec::provider::build_kind; + use trusted_server_core::settings::{IdentityProviderConfig, ProviderSecrets}; + + use super::*; + + fn provider() -> Box { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "unifiedid", + "kind": "unifiedid", + "sources": ["unifiedid.example.com"], + })) + .expect("should parse provider config"); + build_kind::(&config, &ProviderSecrets::default()) + .expect("should build provider") + } + + fn header(cookie: &serde_json::Value) -> String { + format!( + "pbjs-unifiedid={}", + urlencoding::encode(&cookie.to_string()) + ) + } + + #[test] + fn reads_the_tdid_with_prebid_fields() { + let cookie = json!({ + "TDID": "tdid-example", + "TDID_LOOKUP": "TRUE", + "TDID_CREATED_AT": "2026-01-01T00:00:00", + }); + let header = header(&cookie); + + let eids = provider() + .read_request_cookies(RequestCookies::new(&header)) + .expect("should read the TDID"); + + let eid = &eids[0]; + assert_eq!(eid.source, "unifiedid.example.com", "should use the source"); + assert_eq!( + eid.inserter.as_deref(), + Some("adserver.org"), + "should set inserter" + ); + assert_eq!( + eid.matcher.as_deref(), + Some("adserver.org"), + "should set matcher" + ); + assert_eq!(eid.mm, Some(4), "should set mm"); + let uid = &eid.uids[0]; + assert_eq!(uid.id, "tdid-example", "should store only the TDID"); + assert_eq!(uid.atype, Some(1), "should be device-based"); + assert_eq!( + uid.ext.as_deref().map(RawValue::get), + Some(r#"{"rtiPartner":"TDID"}"#), + "should mark the rtiPartner" + ); + } + + #[test] + fn ignores_missing_malformed_and_blank_ids() { + for cookie in [ + json!({}), + json!({ "TDID": "" }), + json!({ "TDID": "0" }), + json!({ "TDID": 5 }), + ] { + assert!( + provider() + .read_request_cookies(RequestCookies::new(&header(&cookie))) + .is_none(), + "should ignore {cookie}" + ); + } + assert!( + provider() + .read_request_cookies(RequestCookies::new("pbjs-unifiedid={broken")) + .is_none(), + "should ignore malformed JSON" + ); + } +} diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-unifiedid.md b/docs/superpowers/plans/2026-11-02-identity-provider-unifiedid.md new file mode 100644 index 000000000..ce53d5fb5 --- /dev/null +++ b/docs/superpowers/plans/2026-11-02-identity-provider-unifiedid.md @@ -0,0 +1,30 @@ +# Unified ID Identity Provider Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Store The Trade Desk ID (TDID) from Prebid's Unified ID first-party cookie, as an `[[ec.identity_providers]]` entry with `kind = "unifiedid"`. No server call. + +**Architecture:** `crates/trusted-server-identity-providers/src/unifiedid.rs` holds `UnifiedId` (a `ProviderKind`), `UnifiedIdSettings` and `UnifiedIdProvider`. The provider implements only `IdentityProvider::read_request_cookies` and keeps EID-level fields (`EidLevelFields::Preserved`). + +**Spec:** `docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md`, section 7. + +## Cookie + +- Prebid's `unifiedIdSystem` stores percent-encoded JSON in `pbjs-unifiedid`: `TDID`, `TDID_LOOKUP`, `TDID_CREATED_AT`. Only `TDID` is read. +- Default `cookie_names = ["pbjs-unifiedid"]`, read in order. +- Malformed JSON, a missing or non-string `TDID`, or a blank or opt-out ID reads as no ID. + +No Lockr capability. + +## Stored ID + +- One EID shaped as Prebid sends it: `atype = 1`, UID `ext` `{"rtiPartner":"TDID"}`, `inserter` and `matcher` `adserver.org`, `mm = 4`. +- Default bounded expiry; each request carrying the cookie refreshes it. +- The source must not also be claimed by a liveintent `thetradedesk` attribute; pick one owner. + +## Tasks + +- [ ] Add `unifiedid.rs`; register the kind in `is_registered_kind` and `build`. +- [ ] Unit tests: TDID with Prebid's EID fields, missing, malformed and blank IDs ignored. +- [ ] Add a commented `kind = "unifiedid"` example to `trusted-server.example.toml`. +- [ ] Run `cargo test-fastly` and `cargo clippy-fastly`. diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 02c5c7a02..36c0b565a 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -276,6 +276,19 @@ pull_sync_concurrency = 3 # sources = ["criteo.example.com"] # [ec.identity_providers.settings] # cookie_names = ["cto_bidid"] +# +# Unified ID (kind "unifiedid"): no lookup and no endpoint. Reads The Trade +# Desk ID from the `TDID` field of Prebid's `pbjs-unifiedid` cookie and stores +# it as Prebid sends it: atype 1, UID `ext` `{"rtiPartner":"TDID"}`, and +# `inserter`/`matcher` `adserver.org` with `mm` 4. Takes exactly one source; +# do not also list that source for a liveintent `thetradedesk` attribute. +# [[ec.identity_providers]] +# id = "unifiedid" +# kind = "unifiedid" +# enabled = false +# sources = ["unifiedid.example.com"] +# [ec.identity_providers.settings] +# cookie_names = ["pbjs-unifiedid"] # ============================================================================= From 4853bec13e0ed993e51119b476a7c453a5ff0b5e Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Mon, 2 Nov 2026 22:00:02 +0530 Subject: [PATCH 21/72] Document that LiveIntent partner IDs need the publisher distributor ID Signed-off-by: dhruv8sh --- .../plans/2026-11-02-identity-provider-liveintent.md | 2 +- trusted-server.example.toml | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md b/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md index 1d38c99c2..378554e7c 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md @@ -34,7 +34,7 @@ No Lockr capability: Lockr AIM is not documented to deliver LiveIntent IDs, and ## Response -- `204 No Content` is `LookupOutcome::NoMatch`. LiveIntent answers `204` to anonymous `gdpr=1` lookups and returns only `nonId` without its own third-party cookie; another non-2xx status or a non-object body is `ProviderError::InvalidResponse`. +- `204 No Content` is `LookupOutcome::NoMatch`. LiveIntent answers `204` to anonymous `gdpr=1` lookups. On the `prebid` path it returns only `nonId`; with the publisher's own `distributor_id` (for example `did-0123`) and the `_lc2_fpi` first-party ID as `duid`, it also returns partner IDs without its third-party cookie, so a publisher wanting partner IDs must set `distributor_id`; another non-2xx status or a non-object body is `ProviderError::InvalidResponse`. - Each resolved attribute with a non-empty string value becomes one EID with `atype = 3`, borrowed from the body; `segments`, the TDID `matcher` and the UID `ext` values are added as above. ## Tasks diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 36c0b565a..565704378 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -191,8 +191,10 @@ pull_sync_concurrency = 3 # `nonId` is LiveIntent's own ID and carries the reply's `segments` as EID # `ext`; the others are partner IDs, sent with UID `ext.provider` naming # LiveIntent (the Trade Desk ID also with `rtiPartner` and EID `matcher`). -# `distributor_id` is optional and must be a LiveIntent-issued `did-...` ID; -# without it the lookup uses the `prebid` path, as Prebid does. +# `distributor_id` must be a LiveIntent-issued `did-...` ID; without it the +# lookup uses the `prebid` path, as Prebid does. Set it to the publisher's own +# distributor ID (its Prebid `liveIntentId` `distributorId`) to get partner +# IDs: on the `prebid` path LiveIntent returns only `nonId`. # [[ec.identity_providers]] # id = "liveintent" # kind = "liveintent" From ab289014fc4fb39c2aebf9ca9e4b5a48e28248cb Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Tue, 3 Nov 2026 20:09:20 +0530 Subject: [PATCH 22/72] Honour vendor cookie opt-outs and bound cookie-supplied expiry Providers now report found IDs, an opt-out or nothing from their first-party cookies. An opt-out skips the provider's lookup as skipped_opted_out and removes its stored sources in the write plan, without a browser fallback. ID5 (universal_uid 0), UID2 (Prebid latestToken optout) and SharedID (_pubcid_optout) report opt-outs. Cookies are read on every consented navigation, so an opt-out or changed ID is noticed while sources hold live UIDs; a cookie repeating the held UIDs is not rewritten, and lookups still respect due status. UID2 parses Prebid's stored-tokens JSON and stores only latestToken.advertising_token, never JSON verbatim. Provider-stated absolute expiry is clamped to 30 days after the request in validation. Adds LiveRamp envelope length tests around the 1024-byte UID limit. Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 190 ++++++++++++++++-- .../src/ec/coordinator/results.rs | 4 +- .../src/ec/coordinator/validate.rs | 31 ++- crates/trusted-server-core/src/ec/provider.rs | 45 ++++- .../src/common.rs | 10 + .../src/connectid.rs | 17 +- .../src/criteo.rs | 19 +- .../src/id5.rs | 77 +++++-- .../src/liveramp.rs | 49 ++++- .../src/sharedid.rs | 35 ++-- .../src/uid2.rs | 179 +++++++++++++++-- .../src/unifiedid.rs | 15 +- 12 files changed, 575 insertions(+), 96 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index d85359da5..9a51a4fb3 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -26,11 +26,11 @@ use crate::ec::EcKvSnapshot; use crate::ec::consent::ec_consent_granted; use crate::ec::eid_types::{ProviderId, SourceDomain, UnixSeconds}; use crate::ec::generation::ec_hash; -use crate::ec::kv_types::KvEntry; +use crate::ec::kv_types::{KvEid, KvEntry}; use crate::ec::provider::{ - ClientSignals, IdentityProvider, LookupInput, LookupOutcome, OwnedEntries, ProviderConfigError, - ProviderEid, ProviderResponse, RequestCookies, SourceInput, destination_allowed, parse_sources, - validate_enabled_configs, + ClientSignals, CookieRead, IdentityProvider, LookupInput, LookupOutcome, OwnedEntries, + ProviderConfigError, ProviderEid, ProviderResponse, RequestCookies, SourceInput, + destination_allowed, parse_sources, validate_enabled_configs, }; use crate::ec::rate_limiter::RateLimiter; use crate::ec::registry::PartnerRegistry; @@ -522,6 +522,37 @@ impl IdentityCoordinator for ProviderCoordinator

{ continue; } let (due, input_blocked) = Self::due_sources(registered, entry, lookup.now); + // The vendor's own first-party cookies are read on every + // navigation, so an opt-out or a changed ID is noticed while the + // sources still hold live UIDs. Their IDs stand in for a lookup. + match registered.provider.read_request_cookies(lookup.cookies) { + CookieRead::OptedOut => { + log::debug!( + "Identity lookup: provider={} opted out in its cookies", + registered.id + ); + pending.early.push(skip(SkipReason::OptedOut)); + continue; + } + CookieRead::Found(eids) => { + let found = self.validated(provider, eids, lookup.now); + if matches!(found.status, LookupStatus::Found { .. }) { + // A cookie repeating the held UIDs is not rewritten, + // so its relative expiry does not cause a write on + // every navigation. + if due.is_empty() && holds_same_uids(entry, &found, lookup.now) { + pending.early.push(skip(SkipReason::NotDue)); + } else { + pending.from_cookies.push(Tagged { + provider, + value: found, + }); + } + continue; + } + } + CookieRead::Absent => {} + } if due.is_empty() { pending.early.push(skip(if input_blocked { SkipReason::Input @@ -530,19 +561,6 @@ impl IdentityCoordinator for ProviderCoordinator

{ })); continue; } - // IDs in the vendor's own first-party cookies stand in for a - // lookup. Only providers with due sources read them, so a held - // live UID is not rewritten on every navigation. - if let Some(eids) = registered.provider.read_request_cookies(lookup.cookies) { - let found = self.validated(provider, eids, lookup.now); - if matches!(found.status, LookupStatus::Found { .. }) { - pending.from_cookies.push(Tagged { - provider, - value: found, - }); - continue; - } - } let input = LookupInput { ec_id: lookup.ec_id, consent: lookup.consent, @@ -659,6 +677,32 @@ impl IdentityCoordinator for ProviderCoordinator

{ } } +/// Whether the row already holds exactly the UIDs `found` read from cookies, +/// for every source `found` returned. +fn holds_same_uids(entry: Option<&KvEntry>, found: &ProviderLookup, now: UnixSeconds) -> bool { + let Some(entry) = entry else { + return false; + }; + found.entries.iter().all(|(source, eids)| { + let Some(record) = entry.ids.get(source.as_str()) else { + return false; + }; + let held: HashSet<&str> = record + .eids() + .iter() + .flat_map(KvEid::uids) + .filter(|uid| uid.is_live(now)) + .map(|uid| uid.id().as_str()) + .collect(); + let read: HashSet<&str> = eids + .iter() + .flat_map(KvEid::uids) + .map(|uid| uid.id().as_str()) + .collect(); + held == read + }) +} + fn log_outcome(lookup: &ProviderLookup) { let (kept, dropped) = match lookup.status { LookupStatus::Found { kept, dropped } => (kept, dropped.total()), @@ -753,9 +797,14 @@ mod tests { fn read_request_cookies<'cookie>( &self, cookies: RequestCookies<'cookie>, - ) -> Option>> { - let id = cookies.get(self.1?)?; - Some(vec![ProviderEid { + ) -> CookieRead<'cookie> { + let Some(id) = self.1.and_then(|name| cookies.get(name)) else { + return CookieRead::Absent; + }; + if id == "optout" { + return CookieRead::OptedOut; + } + CookieRead::Found(vec![ProviderEid { source: Cow::Borrowed("ids.example.com"), inserter: None, matcher: None, @@ -933,6 +982,24 @@ mod tests { snapshot: &EcKvSnapshot, consent: &ConsentContext, limiter: &dyn RateLimiter, + ) -> LookupResults { + run_with_cookies( + coordinator, + transport, + snapshot, + consent, + limiter, + "vendor_id=c-1", + ) + } + + fn run_with_cookies( + coordinator: &ProviderCoordinator, + transport: &CannedTransport, + snapshot: &EcKvSnapshot, + consent: &ConsentContext, + limiter: &dyn RateLimiter, + cookies: &str, ) -> LookupResults { let context = LookupContext { ec_id: EC_ID, @@ -943,7 +1010,7 @@ mod tests { user_agent: None, }, publisher_domain: "www.example.com", - cookies: RequestCookies::new("vendor_id=c-1"), + cookies: RequestCookies::new(cookies), rate_limiter: limiter, now: UnixSeconds::new(1000), }; @@ -1363,6 +1430,87 @@ mod tests { assert!(transport.sent.borrow().is_empty(), "should send no lookup"); } + fn cookie_coordinator() -> ProviderCoordinator { + coordinator_with(RegisteredProvider::new( + provider_config(""), + cookie_provider(SourceInput::Anonymous, Some("vendor_id")), + )) + } + + fn held_snapshot(id: &str) -> EcKvSnapshot { + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry + .ids + .insert("ids.example.com".to_owned(), provider_record(id)); + seeded_snapshot(&entry) + } + + #[test] + fn cookie_opt_out_skips_the_lookup_and_clears_the_sources() { + let coordinator = cookie_coordinator(); + let transport = CannedTransport::new(ok_reply); + + for snapshot in [missing(), held_snapshot("p-1")] { + let results = run_with_cookies( + &coordinator, + &transport, + &snapshot, + &granting(), + &Limiter(false), + "vendor_id=optout", + ); + + let lookup = results.iter().next().expect("should report the provider"); + assert_eq!( + lookup.status, + LookupStatus::Skipped(SkipReason::OptedOut), + "should skip for the opt-out" + ); + assert_eq!( + lookup.cleared, + [SourceDomain::try_from("ids.example.com").expect("should parse")], + "should clear the provider's sources" + ); + } + assert!(transport.sent.borrow().is_empty(), "should send no lookup"); + } + + #[test] + fn changed_cookie_replaces_a_held_live_uid() { + let coordinator = cookie_coordinator(); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &held_snapshot("p-1"), &granting()); + + let lookup = results.iter().next().expect("should report the provider"); + assert!( + matches!(lookup.status, LookupStatus::Found { kept: 1, .. }), + "should read the changed cookie although the source is not due" + ); + assert_eq!( + lookup.entries[0].1[0].uids()[0].id().as_str(), + "c-1", + "should store the new cookie value" + ); + assert!(transport.sent.borrow().is_empty(), "should send no lookup"); + } + + #[test] + fn unchanged_cookie_for_a_held_uid_is_not_rewritten() { + let coordinator = cookie_coordinator(); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &held_snapshot("c-1"), &granting()); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::NotDue), + "should skip a cookie repeating the held UID" + ); + assert!(transport.sent.borrow().is_empty(), "should send no lookup"); + } + #[test] fn missing_or_invalid_cookie_falls_back_to_the_lookup() { let coordinator = coordinator_with(RegisteredProvider::new( diff --git a/crates/trusted-server-core/src/ec/coordinator/results.rs b/crates/trusted-server-core/src/ec/coordinator/results.rs index f5015e452..f52f211c2 100644 --- a/crates/trusted-server-core/src/ec/coordinator/results.rs +++ b/crates/trusted-server-core/src/ec/coordinator/results.rs @@ -63,8 +63,8 @@ pub enum SkipReason { Destination, /// The provider's hourly rate limit is reached. RateLimited, - /// The vendor reports an opt-out; the provider's stored UIDs are - /// removed. + /// The vendor or its cookies report an opt-out; the provider's stored + /// UIDs are removed. OptedOut, } diff --git a/crates/trusted-server-core/src/ec/coordinator/validate.rs b/crates/trusted-server-core/src/ec/coordinator/validate.rs index 2f08be0ee..92203027e 100644 --- a/crates/trusted-server-core/src/ec/coordinator/validate.rs +++ b/crates/trusted-server-core/src/ec/coordinator/validate.rs @@ -13,6 +13,9 @@ use crate::ec::registry::PartnerRegistry; /// Largest private refresh state a provider may keep with one UID. const MAX_REFRESH_STATE_BYTES: usize = 2048; +/// Longest a provider-stated absolute expiry may lie after the request: +/// 30 days. Cookie- and vendor-supplied timestamps are clamped to it. +const MAX_ABSOLUTE_EXPIRY_SECS: u64 = 30 * 24 * 60 * 60; /// Policy one response is validated against. pub(crate) struct ValidationPolicy<'a> { @@ -164,7 +167,7 @@ fn stored_uid( fn resolve_expiry(expiry: ProviderExpiry, now: UnixSeconds) -> Option { match expiry { ProviderExpiry::Unspecified => None, - ProviderExpiry::At(at) => Some(at), + ProviderExpiry::At(at) => Some(at.min(now.saturating_add_secs(MAX_ABSOLUTE_EXPIRY_SECS))), ProviderExpiry::After(after) => Some(now.saturating_add_secs(after.as_secs())), } } @@ -314,6 +317,32 @@ mod tests { "should keep only usable refresh state" ); } + fn clamps_absolute_expiry_to_thirty_days() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + let mut far = uid("u-far"); + far.expiry = ProviderExpiry::At(UnixSeconds::new(u64::MAX)); + let mut near = uid("u-near"); + near.expiry = ProviderExpiry::At(UnixSeconds::new(5000)); + + let (entries, dropped) = validate( + vec![eid("ids.example.com", vec![far, near])], + &policy(&provider, &claimed, &partners), + ); + + assert_eq!(dropped, DropCounts::default(), "should drop nothing"); + let uids = entries[0].1[0].uids(); + assert_eq!( + uids[0].expiry(), + Expiry::At(UnixSeconds::new(1000 + MAX_ABSOLUTE_EXPIRY_SECS)), + "should clamp a far expiry to now plus 30 days" + ); + assert_eq!( + uids[1].expiry(), + Expiry::At(UnixSeconds::new(5000)), + "should keep a near expiry" + ); } #[test] diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs index 006280533..641206301 100644 --- a/crates/trusted-server-core/src/ec/provider.rs +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -73,16 +73,19 @@ pub trait IdentityProvider: Send + Sync { /// Reads IDs from the vendor's own first-party cookies on the request. /// - /// Runs before any lookup for providers with due sources. Returned - /// entries go through the same validation, consent gates and write plan - /// as lookup results, and a kept UID makes the provider skip its lookup - /// for this navigation. The default reads nothing. + /// Runs on every consented navigation, before the due check, so a + /// changed cookie or an opt-out is noticed even while the provider's + /// sources hold live UIDs. Returned entries go through the same + /// validation, consent gates and write plan as lookup results, and a + /// kept UID makes the provider skip its lookup for this navigation. + /// [`CookieRead::OptedOut`] skips the lookup and removes the provider's + /// stored UIDs. The default reads nothing. fn read_request_cookies<'cookie>( &self, cookies: RequestCookies<'cookie>, - ) -> Option>> { + ) -> CookieRead<'cookie> { let _ = cookies; - None + CookieRead::Absent } } @@ -112,11 +115,39 @@ impl IdentityProvider for Box

{ fn read_request_cookies<'cookie>( &self, cookies: RequestCookies<'cookie>, - ) -> Option>> { + ) -> CookieRead<'cookie> { (**self).read_request_cookies(cookies) } } +/// What a provider found in the vendor's own first-party cookies. +#[derive(Debug)] +pub enum CookieRead<'cookie> { + /// The cookies hold IDs. + Found(Vec>), + /// The cookies record that the user opted out of the vendor. + OptedOut, + /// The cookies hold no usable ID and no opt-out. + Absent, +} + +impl<'cookie> CookieRead<'cookie> { + /// The IDs found, or `None` for an opt-out or no ID. + #[must_use] + pub fn into_found(self) -> Option>> { + match self { + Self::Found(eids) => Some(eids), + Self::OptedOut | Self::Absent => None, + } + } +} + +impl<'cookie> From>>> for CookieRead<'cookie> { + fn from(eids: Option>>) -> Self { + eids.map_or(Self::Absent, Self::Found) + } +} + /// The request's `Cookie` header, lent to providers that read their own /// first-party cookies. #[derive(Debug, Clone, Copy, Default)] diff --git a/crates/trusted-server-identity-providers/src/common.rs b/crates/trusted-server-identity-providers/src/common.rs index c034c0f49..0b510ed30 100644 --- a/crates/trusted-server-identity-providers/src/common.rs +++ b/crates/trusted-server-identity-providers/src/common.rs @@ -291,6 +291,16 @@ impl CookieReader { self.own.is_empty() && self.lockr.is_empty() } + /// Whether any configured cookie, own or Lockr, has a value matching + /// `matches`. + pub(crate) fn any(&self, cookies: RequestCookies<'_>, matches: impl Fn(&str) -> bool) -> bool { + self.own + .iter() + .chain(self.lockr.iter()) + .filter_map(|name| cookies.get(name)) + .any(matches) + } + /// Returns the first value `parse` accepts, trying the own cookies /// before the Lockr cookies. pub(crate) fn read<'cookie, T>( diff --git a/crates/trusted-server-identity-providers/src/connectid.rs b/crates/trusted-server-identity-providers/src/connectid.rs index 8013639f3..c6823a8c9 100644 --- a/crates/trusted-server-identity-providers/src/connectid.rs +++ b/crates/trusted-server-identity-providers/src/connectid.rs @@ -22,8 +22,8 @@ use error_stack::Report; use serde::Deserialize; use trusted_server_core::ec::eid_types::{SourceDomain, UnixSeconds}; use trusted_server_core::ec::provider::{ - IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, - ProviderConfigError, ProviderEid, ProviderError, ProviderExpiry, ProviderKind, ProviderRequest, + CookieRead, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, + ProviderConfigError, ProviderError, ProviderExpiry, ProviderKind, ProviderRequest, ProviderResponse, RequestCookies, }; @@ -136,9 +136,11 @@ impl IdentityProvider for ConnectIdProvider { fn read_request_cookies<'cookie>( &self, cookies: RequestCookies<'cookie>, - ) -> Option>> { - let (id, expiry) = self.cookies.read(cookies, connect_id)?; - Some(vec![cookie_eid(&self.source, Cow::Owned(id), expiry)]) + ) -> CookieRead<'cookie> { + let Some((id, expiry)) = self.cookies.read(cookies, connect_id) else { + return CookieRead::Absent; + }; + CookieRead::Found(vec![cookie_eid(&self.source, Cow::Owned(id), expiry)]) } } @@ -163,7 +165,9 @@ mod tests { fn read(cookie: &serde_json::Value) -> Option<(String, ProviderExpiry)> { let header = format!("connectId={}", urlencoding::encode(&cookie.to_string())); - let eids = provider().read_request_cookies(RequestCookies::new(&header))?; + let eids = provider() + .read_request_cookies(RequestCookies::new(&header)) + .into_found()?; assert_eq!( eids[0].source, "connectid.example.com", "should use the source" @@ -237,6 +241,7 @@ mod tests { assert!( provider() .read_request_cookies(RequestCookies::new("connectId={broken")) + .into_found() .is_none(), "should ignore malformed JSON" ); diff --git a/crates/trusted-server-identity-providers/src/criteo.rs b/crates/trusted-server-identity-providers/src/criteo.rs index 21ff79cf6..8079f9e6e 100644 --- a/crates/trusted-server-identity-providers/src/criteo.rs +++ b/crates/trusted-server-identity-providers/src/criteo.rs @@ -11,9 +11,9 @@ use error_stack::Report; use serde::Deserialize; use trusted_server_core::ec::eid_types::SourceDomain; use trusted_server_core::ec::provider::{ - IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, - ProviderConfigError, ProviderEid, ProviderError, ProviderKind, ProviderRequest, - ProviderResponse, RequestCookies, + CookieRead, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, + ProviderConfigError, ProviderError, ProviderKind, ProviderRequest, ProviderResponse, + RequestCookies, }; use crate::common::{CookieReader, bounded_expiry, cookie_capabilities, device_eid, is_usable_id}; @@ -88,11 +88,14 @@ impl IdentityProvider for CriteoProvider { fn read_request_cookies<'cookie>( &self, cookies: RequestCookies<'cookie>, - ) -> Option>> { - let id = self + ) -> CookieRead<'cookie> { + let Some(id) = self .cookies - .read(cookies, |value| is_usable_id(value).then_some(value.trim()))?; - Some(vec![device_eid( + .read(cookies, |value| is_usable_id(value).then_some(value.trim())) + else { + return CookieRead::Absent; + }; + CookieRead::Found(vec![device_eid( &self.source, Cow::Borrowed(id), bounded_expiry(None), @@ -122,6 +125,7 @@ mod tests { fn stores_the_bid_id_verbatim() { let eids = provider() .read_request_cookies(RequestCookies::new("cto_bidid=bidid-example%3D%3D")) + .into_found() .expect("should read the bid ID"); assert_eq!( @@ -142,6 +146,7 @@ mod tests { assert!( provider() .read_request_cookies(RequestCookies::new(header)) + .into_found() .is_none(), "should ignore {header}" ); diff --git a/crates/trusted-server-identity-providers/src/id5.rs b/crates/trusted-server-identity-providers/src/id5.rs index 9acefcac5..c0c9e5064 100644 --- a/crates/trusted-server-identity-providers/src/id5.rs +++ b/crates/trusted-server-identity-providers/src/id5.rs @@ -21,7 +21,10 @@ //! and the default TTL. The response `signature` is never stored. A //! percent-encoded JSON cookie is decoded and its entries, `ext` objects //! included, are copied out of the decoded text. A cookie that yields no -//! usable UID is ignored and the lookup runs as usual. +//! usable UID is ignored and the lookup runs as usual. A cookie holding +//! ID5's opt-out (a bare `0`, or a reply whose `universal_uid` is `0`) +//! reports an opt-out: the lookup is skipped and stored ID5 UIDs are +//! removed. //! //! The request field names for the client IP and user agent follow ID5's //! server-to-server integration and should be confirmed with ID5 for each @@ -39,9 +42,10 @@ use http::{HeaderMap, HeaderValue, Method}; use serde::{Deserialize, Serialize}; use trusted_server_core::ec::eid_types::SourceDomain; use trusted_server_core::ec::provider::{ - EidLevelFields, EidShape, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, - ProviderCommon, ProviderConfigError, ProviderEid, ProviderError, ProviderKind, ProviderRequest, - ProviderResponse, ProviderUid, RequestCookies, SourceCapability, SourceInput, + CookieRead, EidLevelFields, EidShape, IdentityProvider, LookupInput, LookupOutcome, + ProviderCapabilities, ProviderCommon, ProviderConfigError, ProviderEid, ProviderError, + ProviderKind, ProviderRequest, ProviderResponse, ProviderUid, RequestCookies, SourceCapability, + SourceInput, }; use url::Url; @@ -380,9 +384,34 @@ impl IdentityProvider for Id5Provider { fn read_request_cookies<'cookie>( &self, cookies: RequestCookies<'cookie>, - ) -> Option>> { - self.cookies.read(cookies, |value| self.cookie_eids(value)) + ) -> CookieRead<'cookie> { + if self.cookies.any(cookies, is_opt_out_cookie) { + return CookieRead::OptedOut; + } + CookieRead::from(self.cookies.read(cookies, |value| self.cookie_eids(value))) + } +} + +/// The ID5 opt-out UID. +const OPT_OUT_UID: &str = "0"; + +/// Whether one ID5 cookie value records an opt-out: a bare `0`, or a reply +/// whose `universal_uid` is `0`. +fn is_opt_out_cookie(value: &str) -> bool { + let decoded = percent_decoded(value); + let decoded = decoded.trim(); + if !decoded.starts_with('{') { + return decoded == OPT_OUT_UID; } + serde_json::from_str::(decoded) + .ok() + .and_then(|reply| { + reply + .get("universal_uid") + .and_then(serde_json::Value::as_str) + .map(|uid| uid == OPT_OUT_UID) + }) + .unwrap_or(false) } /// Copies an EID parsed from a temporary string so it outlives that string. @@ -664,6 +693,7 @@ mod tests { fn read(provider: &dyn IdentityProvider, header: &str) -> Option> { provider .read_request_cookies(RequestCookies::new(header)) + .into_found() .map(|eids| { eids.into_iter() .flat_map(|eid| { @@ -750,7 +780,7 @@ mod tests { let provider = build_kind::(&config, &ProviderSecrets::default()).expect("should build provider"); - let uids = read(&provider, "id5id=0; example_lockr_id5=bare-example") + let uids = read(&provider, "id5id=%7Bbroken; example_lockr_id5=bare-example") .expect("should read the bare ID"); assert_eq!( @@ -787,8 +817,11 @@ mod tests { &provider, "example_lockr_id5=lockr-example; id5id=own-example", ); - let lockr = read(&provider, "id5id=0; example_lockr_id5=lockr-example"); - let neither = read(&provider, "example_lockr_id5=0"); + let lockr = read( + &provider, + "id5id=%7Bbroken; example_lockr_id5=lockr-example", + ); + let neither = read(&provider, "example_lockr_id5=%7Bbroken"); assert_eq!( own_first.map(|uids| uids[0].1.clone()).as_deref(), @@ -804,7 +837,7 @@ mod tests { } #[test] - fn ignores_opted_out_and_malformed_cookies_so_the_lookup_runs() { + fn reports_an_opt_out_cookie_so_the_lookup_is_skipped() { let provider = provider(); let opted_out = json!({ "universal_uid": "0", "ids": {} }).to_string(); @@ -812,12 +845,28 @@ mod tests { "id5id=0".to_owned(), format!("id5id={opted_out}"), format!("id5id={}", urlencoding::encode(&opted_out)), - "id5id={not json".to_owned(), - "other=bare-example".to_owned(), ] { assert!( - read(&*provider, &header).is_none(), - "should read no ID from {header}" + matches!( + provider.read_request_cookies(RequestCookies::new(&header)), + CookieRead::OptedOut + ), + "should report an opt-out from {header}" + ); + } + } + + #[test] + fn ignores_malformed_cookies_so_the_lookup_runs() { + let provider = provider(); + + for header in ["id5id={not json", "other=bare-example"] { + assert!( + matches!( + provider.read_request_cookies(RequestCookies::new(header)), + CookieRead::Absent + ), + "should read nothing from {header}" ); } } diff --git a/crates/trusted-server-identity-providers/src/liveramp.rs b/crates/trusted-server-identity-providers/src/liveramp.rs index 13182e53d..5b2cb594e 100644 --- a/crates/trusted-server-identity-providers/src/liveramp.rs +++ b/crates/trusted-server-identity-providers/src/liveramp.rs @@ -26,9 +26,9 @@ use error_stack::Report; use serde::Deserialize; use trusted_server_core::ec::eid_types::SourceDomain; use trusted_server_core::ec::provider::{ - IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, - ProviderConfigError, ProviderEid, ProviderError, ProviderKind, ProviderRequest, - ProviderResponse, RequestCookies, + CookieRead, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, + ProviderConfigError, ProviderError, ProviderKind, ProviderRequest, ProviderResponse, + RequestCookies, }; use crate::common::{ @@ -152,9 +152,11 @@ impl IdentityProvider for LiveRampProvider { fn read_request_cookies<'cookie>( &self, cookies: RequestCookies<'cookie>, - ) -> Option>> { - let envelope = self.cookies.read(cookies, envelope)?; - Some(vec![cookie_eid( + ) -> CookieRead<'cookie> { + let Some(envelope) = self.cookies.read(cookies, envelope) else { + return CookieRead::Absent; + }; + CookieRead::Found(vec![cookie_eid( &self.source, Cow::Owned(envelope), bounded_expiry(Some(ENVELOPE_TTL)), @@ -166,6 +168,8 @@ impl IdentityProvider for LiveRampProvider { mod tests { use serde_json::json; use trusted_server_core::consent::types::ConsentContext; + use trusted_server_core::ec::eid_types::{EidValidationError, UidValue}; + use trusted_server_core::ec::kv_types::MAX_UID_LENGTH; use trusted_server_core::ec::provider::{ ClientSignals, OwnedEntries, ProviderExpiry, build_kind, }; @@ -189,7 +193,9 @@ mod tests { } fn read(provider: &dyn IdentityProvider, header: &str) -> Option<(String, ProviderExpiry)> { - let eids = provider.read_request_cookies(RequestCookies::new(header))?; + let eids = provider + .read_request_cookies(RequestCookies::new(header)) + .into_found()?; assert_eq!(eids.len(), 1, "should return one EID"); assert_eq!( eids[0].source, "liveramp.example.com", @@ -220,6 +226,35 @@ mod tests { } } + #[test] + fn collects_envelopes_up_to_the_uid_length_limit_and_drops_longer_ones() { + let provider = provider(&json!({})); + + for len in [703, 1000, MAX_UID_LENGTH] { + let raw = "A".repeat(len); + let ats = encoded(&json!({ "envelope": raw }), &STANDARD); + for header in [format!("idl_env={raw}"), format!("_lr_env={ats}")] { + let (id, _) = read(&*provider, &header).expect("should read the envelope"); + + assert_eq!(id.len(), len, "should keep the whole envelope"); + assert!( + UidValue::try_from(id).is_ok(), + "should pass validation at {len} bytes" + ); + } + } + + let (id, _) = read( + &*provider, + &format!("idl_env={}", "A".repeat(MAX_UID_LENGTH + 1)), + ) + .expect("should read the oversized envelope"); + assert!( + matches!(UidValue::try_from(id), Err(EidValidationError::UidTooLong)), + "should drop an envelope over the UID length limit in validation" + ); + } + #[test] fn reads_a_bare_prebid_idl_env_envelope() { let provider = provider(&json!({})); diff --git a/crates/trusted-server-identity-providers/src/sharedid.rs b/crates/trusted-server-identity-providers/src/sharedid.rs index f53e1b97d..0835e6ecb 100644 --- a/crates/trusted-server-identity-providers/src/sharedid.rs +++ b/crates/trusted-server-identity-providers/src/sharedid.rs @@ -3,7 +3,8 @@ //! //! Prebid stores the ID as a bare UUID in `_sharedid` (the publisher's //! configured name) or the legacy `_pubcid`. It is sent as a device-based -//! (agent type 1) EID. The opt-out cookie `_pubcid_optout` suppresses it. +//! (agent type 1) EID. The opt-out cookie `_pubcid_optout` reports an +//! opt-out, which removes a stored `SharedID`. //! No server call is made. use std::borrow::Cow; @@ -12,9 +13,9 @@ use error_stack::Report; use serde::Deserialize; use trusted_server_core::ec::eid_types::SourceDomain; use trusted_server_core::ec::provider::{ - IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, - ProviderConfigError, ProviderEid, ProviderError, ProviderKind, ProviderRequest, - ProviderResponse, RequestCookies, + CookieRead, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, + ProviderConfigError, ProviderError, ProviderKind, ProviderRequest, ProviderResponse, + RequestCookies, }; use crate::common::{CookieReader, bounded_expiry, cookie_capabilities, device_eid, is_usable_id}; @@ -92,14 +93,17 @@ impl IdentityProvider for SharedIdProvider { fn read_request_cookies<'cookie>( &self, cookies: RequestCookies<'cookie>, - ) -> Option>> { + ) -> CookieRead<'cookie> { if cookies.get(OPT_OUT_COOKIE).is_some() { - return None; + return CookieRead::OptedOut; } - let id = self + let Some(id) = self .cookies - .read(cookies, |value| is_usable_id(value).then_some(value.trim()))?; - Some(vec![device_eid( + .read(cookies, |value| is_usable_id(value).then_some(value.trim())) + else { + return CookieRead::Absent; + }; + CookieRead::Found(vec![device_eid( &self.source, Cow::Borrowed(id), bounded_expiry(None), @@ -126,7 +130,9 @@ mod tests { } fn read(header: &str) -> Option { - let eids = provider().read_request_cookies(RequestCookies::new(header))?; + let eids = provider() + .read_request_cookies(RequestCookies::new(header)) + .into_found()?; assert_eq!( eids[0].source, "sharedid.example.com", "should use the source" @@ -153,8 +159,13 @@ mod tests { #[test] fn honours_opt_out_and_ignores_blank_ids() { assert!( - read("_sharedid=sharedid-example; _pubcid_optout=1").is_none(), - "should honour the opt-out cookie" + matches!( + provider().read_request_cookies(RequestCookies::new( + "_sharedid=sharedid-example; _pubcid_optout=1" + )), + CookieRead::OptedOut + ), + "should report the opt-out cookie" ); for header in ["_sharedid=", "_sharedid=0", "other=1"] { assert!(read(header).is_none(), "should ignore {header}"); diff --git a/crates/trusted-server-identity-providers/src/uid2.rs b/crates/trusted-server-identity-providers/src/uid2.rs index 6f0a137ac..7ff1cd6e6 100644 --- a/crates/trusted-server-identity-providers/src/uid2.rs +++ b/crates/trusted-server-identity-providers/src/uid2.rs @@ -9,14 +9,22 @@ //! never stored, and no refresh is made server side. //! //! When `__uid_2` yields nothing, the cookies in `cookie_names` (default -//! Prebid's `__uid2_advertising_token`) are read as a raw token with the +//! Prebid's `__uid2_advertising_token`) are read. Prebid's UID2 module +//! stores percent-encoded JSON there, `{"originalToken":…,"latestToken":…}`: +//! only `latestToken.advertising_token` is stored, until +//! `latestToken.identity_expires` (milliseconds) or with the shared default +//! TTL when that is missing. Any other value is read as a raw token with the //! shared default TTL. With the Lockr capability enabled //! (`[settings.lockr]`), the cookies Lockr AIM writes the raw token into //! (default key `__uid2_advertising_token`) are read last, with the same //! parser. //! +//! A `latestToken` of `"optout"` in any of those cookies reports an opt-out: +//! no lookup is made and stored UID2 UIDs are removed. +//! //! Parsing is lenient: an undecodable cookie, a missing token or expiry, -//! or a blank or opt-out token reads as no ID. +//! or a blank or opt-out token reads as no ID. A JSON value is never stored +//! as a token. use std::borrow::Cow; @@ -24,8 +32,8 @@ use error_stack::Report; use serde::Deserialize; use trusted_server_core::ec::eid_types::{SourceDomain, UnixSeconds}; use trusted_server_core::ec::provider::{ - IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, - ProviderConfigError, ProviderEid, ProviderError, ProviderExpiry, ProviderKind, ProviderRequest, + CookieRead, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, + ProviderConfigError, ProviderError, ProviderExpiry, ProviderKind, ProviderRequest, ProviderResponse, RequestCookies, }; @@ -117,11 +125,59 @@ fn identity(value: &str) -> Option<(String, ProviderExpiry)> { Some((cookie.advertising_token, ProviderExpiry::At(expires))) } -/// Decodes one raw advertising token cookie. -fn raw_token(value: &str) -> Option<(String, ProviderExpiry)> { - let token = percent_decoded(value); - let token = token.trim(); - is_usable_id(token).then(|| (token.to_owned(), bounded_expiry(None))) +/// `latestToken` value Prebid stores after an opt-out. +const PREBID_OPT_OUT: &str = "optout"; + +/// The fields of Prebid's stored tokens this provider reads. +#[derive(Deserialize)] +struct PrebidTokens { + #[serde(rename = "latestToken")] + latest_token: serde_json::Value, +} + +/// The fields of Prebid's `latestToken` identity this provider reads. +#[derive(Deserialize)] +struct PrebidIdentity { + advertising_token: String, + #[serde(default)] + identity_expires: Option, +} + +/// One token cookie's reading. +enum TokenCookie { + /// A usable token and its expiry. + Token(String, ProviderExpiry), + /// Prebid recorded an opt-out. + OptedOut, +} + +/// Decodes one token cookie: Prebid's stored tokens JSON or a raw token. +/// +/// A JSON value is never taken as a raw token. +fn token_cookie(value: &str) -> Option { + let decoded = percent_decoded(value); + let decoded = decoded.trim(); + if decoded.starts_with('{') { + return prebid_tokens(decoded); + } + is_usable_id(decoded).then(|| TokenCookie::Token(decoded.to_owned(), bounded_expiry(None))) +} + +/// Reads `latestToken` from Prebid's stored tokens JSON. +fn prebid_tokens(json: &str) -> Option { + let tokens: PrebidTokens = serde_json::from_str(json).ok()?; + if tokens.latest_token.as_str() == Some(PREBID_OPT_OUT) { + return Some(TokenCookie::OptedOut); + } + let identity: PrebidIdentity = serde_json::from_value(tokens.latest_token).ok()?; + if !is_usable_id(&identity.advertising_token) { + return None; + } + let expiry = identity.identity_expires.map_or_else( + || bounded_expiry(None), + |ms| ProviderExpiry::At(UnixSeconds::new(ms / 1000)), + ); + Some(TokenCookie::Token(identity.advertising_token, expiry)) } impl IdentityProvider for Uid2Provider { @@ -146,12 +202,27 @@ impl IdentityProvider for Uid2Provider { fn read_request_cookies<'cookie>( &self, cookies: RequestCookies<'cookie>, - ) -> Option>> { - let (token, expiry) = cookies + ) -> CookieRead<'cookie> { + let opted_out = self.cookies.any(cookies, |value| { + matches!(token_cookie(value), Some(TokenCookie::OptedOut)) + }); + if opted_out { + return CookieRead::OptedOut; + } + let token = cookies .get(&self.identity_cookie_name) .and_then(identity) - .or_else(|| self.cookies.read(cookies, raw_token))?; - Some(vec![cookie_eid(&self.source, Cow::Owned(token), expiry)]) + .or_else(|| { + self.cookies + .read(cookies, |value| match token_cookie(value)? { + TokenCookie::Token(token, expiry) => Some((token, expiry)), + TokenCookie::OptedOut => None, + }) + }); + let Some((token, expiry)) = token else { + return CookieRead::Absent; + }; + CookieRead::Found(vec![cookie_eid(&self.source, Cow::Owned(token), expiry)]) } } @@ -175,7 +246,9 @@ mod tests { } fn read(provider: &dyn IdentityProvider, header: &str) -> Option<(String, ProviderExpiry)> { - let eids = provider.read_request_cookies(RequestCookies::new(header))?; + let eids = provider + .read_request_cookies(RequestCookies::new(header)) + .into_found()?; assert_eq!(eids[0].source, "uid2.example.com", "should use the source"); let uid = &eids[0].uids[0]; assert_eq!(uid.atype, Some(3), "should be person-based"); @@ -222,6 +295,84 @@ mod tests { ); } + fn prebid_cookie(tokens: &serde_json::Value) -> String { + format!( + "__uid2_advertising_token={}", + urlencoding::encode(&tokens.to_string()) + ) + } + + #[test] + fn reads_the_latest_token_from_prebid_stored_tokens() { + let header = prebid_cookie(&json!({ + "originalToken": { + "advertising_token": "AgAAAAoriginaltoken", + "identity_expires": 1_600_000_000_000_u64, + }, + "latestToken": { + "advertising_token": "AgAAAAlatesttoken", + "refresh_token": "example-refresh-token", + "refresh_response_key": "example-response-key", + "identity_expires": 1_700_000_000_456_u64, + "refresh_from": 1_699_990_000_000_u64, + "refresh_expires": 1_702_000_000_000_u64, + }, + })); + + let (token, expiry) = read(&*provider(&json!({})), &header).expect("should read"); + + assert_eq!( + token, "AgAAAAlatesttoken", + "should store only the latest advertising token" + ); + assert!( + matches!(expiry, ProviderExpiry::At(at) if at.get() == 1_700_000_000), + "should expire at the latest identity_expires" + ); + } + + #[test] + fn reports_a_prebid_opt_out() { + let provider = provider(&json!({})); + let own = identity_cookie(&json!({ + "advertising_token": "AgAAAAowntoken", + "identity_expires": 1_700_000_000_000_u64, + })); + let opt_out = prebid_cookie(&json!({ + "originalToken": { "advertising_token": "AgAAAAoriginaltoken" }, + "latestToken": "optout", + })); + + let header = format!("{own}; {opt_out}"); + + let read = provider.read_request_cookies(RequestCookies::new(&header)); + + assert!( + matches!(read, CookieRead::OptedOut), + "should report the opt-out over any token" + ); + } + + #[test] + fn never_stores_json_as_a_token() { + let provider = provider(&json!({})); + for header in [ + "__uid2_advertising_token=%7Bbroken".to_owned(), + "__uid2_advertising_token={\"latestToken\":{}}".to_owned(), + prebid_cookie(&json!({ "originalToken": {} })), + prebid_cookie(&json!({ "latestToken": { "advertising_token": "" } })), + prebid_cookie(&json!({ "latestToken": 7 })), + ] { + assert!( + matches!( + provider.read_request_cookies(RequestCookies::new(&header)), + CookieRead::Absent + ), + "should read nothing from {header:?}" + ); + } + } + #[test] fn reads_configured_cookie_names() { let provider = provider(&json!({ "cookie_names": ["example_lockr_uid2"] })); diff --git a/crates/trusted-server-identity-providers/src/unifiedid.rs b/crates/trusted-server-identity-providers/src/unifiedid.rs index 56409eb8c..9e065fe43 100644 --- a/crates/trusted-server-identity-providers/src/unifiedid.rs +++ b/crates/trusted-server-identity-providers/src/unifiedid.rs @@ -13,8 +13,8 @@ use serde::Deserialize; use serde_json::value::RawValue; use trusted_server_core::ec::eid_types::SourceDomain; use trusted_server_core::ec::provider::{ - EidLevelFields, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, - ProviderCommon, ProviderConfigError, ProviderEid, ProviderError, ProviderKind, ProviderRequest, + CookieRead, EidLevelFields, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, + ProviderCommon, ProviderConfigError, ProviderError, ProviderKind, ProviderRequest, ProviderResponse, RequestCookies, }; @@ -119,8 +119,10 @@ impl IdentityProvider for UnifiedIdProvider { fn read_request_cookies<'cookie>( &self, cookies: RequestCookies<'cookie>, - ) -> Option>> { - let id = self.cookies.read(cookies, tdid)?; + ) -> CookieRead<'cookie> { + let Some(id) = self.cookies.read(cookies, tdid) else { + return CookieRead::Absent; + }; let mut eid = device_eid(&self.source, Cow::Owned(id), bounded_expiry(None)); eid.inserter = Some(Cow::Borrowed(TDID_INSERTER)); eid.matcher = Some(Cow::Borrowed(TDID_INSERTER)); @@ -128,7 +130,7 @@ impl IdentityProvider for UnifiedIdProvider { for uid in &mut eid.uids { uid.ext = Some(Cow::Borrowed(self.uid_ext)); } - Some(vec![eid]) + CookieRead::Found(vec![eid]) } } @@ -169,6 +171,7 @@ mod tests { let eids = provider() .read_request_cookies(RequestCookies::new(&header)) + .into_found() .expect("should read the TDID"); let eid = &eids[0]; @@ -205,6 +208,7 @@ mod tests { assert!( provider() .read_request_cookies(RequestCookies::new(&header(&cookie))) + .into_found() .is_none(), "should ignore {cookie}" ); @@ -212,6 +216,7 @@ mod tests { assert!( provider() .read_request_cookies(RequestCookies::new("pbjs-unifiedid={broken")) + .into_found() .is_none(), "should ignore malformed JSON" ); From d5a03a71bcfd9eb51313ebd1bc2673d8d34e15e8 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 01:28:44 +0530 Subject: [PATCH 23/72] Refresh UID2 tokens, generate SharedIDs and look up LiveRamp envelopes server side UID2 refreshes its stored advertising token with the stored refresh token and decrypts the AES-256-GCM reply; an optout reply removes the token. SharedID generates a random UUID v4 into KV only, kept for 30 days, when no cookie or stored ID exists. LiveRamp, when given an endpoint, placement ID and Custom ID namespace, looks up a RampID envelope by Custom ID built from the stable EC ID; a 204 reply is an opt-out. Port the after-response collection, refresh-state, opt-out and rate_limit_per_hour framework changes, and update the vendor plans and operator docs. Signed-off-by: dhruv8sh --- Cargo.lock | 70 ++- Cargo.toml | 1 + .../Cargo.toml | 2 + .../src/common.rs | 2 + .../src/id5.rs | 6 + .../src/lib.rs | 131 ++++- .../src/liveintent.rs | 5 + .../src/liveramp.rs | 363 +++++++++++++- .../src/sharedid.rs | 65 ++- .../src/uid2.rs | 457 +++++++++++++++++- docs/guide/api-reference.md | 1 + docs/guide/configuration.md | 36 +- .../2026-11-02-identity-provider-connectid.md | 2 + .../2026-11-02-identity-provider-criteo.md | 6 +- .../plans/2026-11-02-identity-provider-id5.md | 2 + ...2026-11-02-identity-provider-liveintent.md | 2 + .../2026-11-02-identity-provider-liveramp.md | 10 +- ...1-02-identity-provider-lockr-capability.md | 2 + .../2026-11-02-identity-provider-sharedid.md | 10 +- .../2026-11-02-identity-provider-uid2.md | 15 +- .../2026-11-02-identity-provider-unifiedid.md | 8 +- trusted-server.example.toml | 53 +- 22 files changed, 1148 insertions(+), 101 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 554040b1f..a62c86afe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,6 +18,31 @@ dependencies = [ "inout", ] +[[package]] +name = "aes" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fc36d7440d015b45468ba08a21e80e056ebbd014c611c6298bb7503e04bd1" +dependencies = [ + "cipher", + "cpubits", + "cpufeatures 0.3.0", +] + +[[package]] +name = "aes-gcm" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2b8006a0c83f52b62ba44a97b58bf76fe2f70a329e588f67f89691d93d498f" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ctutils", + "ghash", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -976,6 +1001,12 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -1126,6 +1157,15 @@ dependencies = [ "linktime-proc-macro", ] +[[package]] +name = "ctr" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" +dependencies = [ + "cipher", +] + [[package]] name = "ctutils" version = "0.4.3" @@ -2214,6 +2254,15 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "ghash" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" +dependencies = [ + "polyval", +] + [[package]] name = "gix" version = "0.83.0" @@ -2594,7 +2643,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c91d8cffac8849493a82233811bd02b2b183b8cf39bf704de0fa0841b737595" dependencies = [ "bstr", - "hashbrown 0.16.1", + "hashbrown 0.17.1", ] [[package]] @@ -4629,6 +4678,17 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "polyval" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" +dependencies = [ + "cpubits", + "cpufeatures 0.3.0", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.13.1" @@ -4790,7 +4850,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck", - "itertools 0.10.5", + "itertools 0.13.0", "log", "multimap", "once_cell", @@ -4810,7 +4870,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.118", @@ -4823,7 +4883,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.118", @@ -6742,6 +6802,7 @@ dependencies = [ name = "trusted-server-identity-providers" version = "0.1.0" dependencies = [ + "aes-gcm", "base64", "bytes", "error-stack", @@ -6754,6 +6815,7 @@ dependencies = [ "trusted-server-core", "url", "urlencoding", + "uuid", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f43101230..0c5542a45 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ publish = false version = "0.1.0" [workspace.dependencies] +aes-gcm = { version = "0.11", default-features = false, features = ["aes", "alloc"] } anyhow = "1" assert_cmd = "2" async-stream = "0.3" diff --git a/crates/trusted-server-identity-providers/Cargo.toml b/crates/trusted-server-identity-providers/Cargo.toml index 5cfe89243..da9e6a073 100644 --- a/crates/trusted-server-identity-providers/Cargo.toml +++ b/crates/trusted-server-identity-providers/Cargo.toml @@ -14,6 +14,7 @@ doctest = false workspace = true [dependencies] +aes-gcm = { workspace = true } base64 = { workspace = true } bytes = { workspace = true } error-stack = { workspace = true } @@ -25,6 +26,7 @@ serde_json = { workspace = true, features = ["raw_value"] } trusted-server-core = { workspace = true } url = { workspace = true } urlencoding = { workspace = true } +uuid = { workspace = true } [dev-dependencies] toml = { workspace = true } diff --git a/crates/trusted-server-identity-providers/src/common.rs b/crates/trusted-server-identity-providers/src/common.rs index 0b510ed30..21b38171a 100644 --- a/crates/trusted-server-identity-providers/src/common.rs +++ b/crates/trusted-server-identity-providers/src/common.rs @@ -175,6 +175,7 @@ pub(crate) fn cookie_eid<'cookie>( atype: Some(COOKIE_ATYPE), ext: None, expiry, + refresh: None, }], } } @@ -376,6 +377,7 @@ impl<'body> VendorEid<'body> { atype: uid.atype, ext: uid.ext.map(Cow::Borrowed), expiry, + refresh: None, }) .collect(), } diff --git a/crates/trusted-server-identity-providers/src/id5.rs b/crates/trusted-server-identity-providers/src/id5.rs index c0c9e5064..f63e37509 100644 --- a/crates/trusted-server-identity-providers/src/id5.rs +++ b/crates/trusted-server-identity-providers/src/id5.rs @@ -277,6 +277,7 @@ impl Id5Provider { atype: Some(1), ext: link_type.map(Cow::Owned), expiry, + refresh: None, }], }); } @@ -311,6 +312,7 @@ impl Id5Provider { atype: Some(1), ext: None, expiry: bounded_expiry(None), + refresh: None, }], }] } @@ -360,6 +362,7 @@ impl IdentityProvider for Id5Provider { url: self.url.clone(), headers, body: Bytes::from(body), + context: None, }) } @@ -430,6 +433,7 @@ fn owned<'any>(eid: ProviderEid<'_>) -> ProviderEid<'any> { atype: uid.atype, ext: uid.ext.map(|ext| Cow::Owned(ext.into_owned())), expiry: uid.expiry, + refresh: None, }) .collect(), } @@ -475,6 +479,7 @@ mod tests { status: StatusCode::OK, headers, body, + context: None, }) .expect("should parse response") } @@ -678,6 +683,7 @@ mod tests { status, headers: &headers, body, + context: None, }) .expect_err("should reject the response"); assert!( diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index 0b5caaae3..e6cb80a11 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -107,9 +107,8 @@ pub fn build( /// /// # Errors /// -/// Returns [`ProviderConfigError`] when a provider cannot be built or the -/// enabled providers, taken together, fail the coordinator's configuration -/// checks. +/// Returns [`ProviderConfigError`] when a provider cannot be built or fails +/// the coordinator's configuration checks. pub fn build_coordinator( settings: &Settings, ) -> Result, Report> { @@ -189,6 +188,10 @@ mod tests { use serde_json::json; use super::*; + use std::num::NonZeroU32; + + use trusted_server_core::ec::provider::validate_enabled_configs; + use trusted_server_core::settings::IdentityProviderConfig; fn create_test_settings() -> Settings { Settings::from_toml( @@ -338,36 +341,130 @@ mod tests { ); } - #[test] - fn rejects_an_unknown_collect_timing() { - toml::from_str::( - "id = \"example-provider\"\nkind = \"example-kind\"\ncollect = \"later\"", - ) - .expect_err("should reject an unknown `collect` value"); + fn settings_with_cookie_providers(sharedid_collect: &str, criteo_collect: &str) -> Settings { + let mut settings = create_test_settings(); + settings.ec.partners = ["pubcid.org", "criteo.com"] + .iter() + .map(|source| { + toml::from_str(&format!( + "name = \"{source}\"\nsource_domain = \"{source}\"\nbidstream_enabled = true" + )) + .expect("should parse partner") + }) + .collect(); + settings.ec.identity_providers = vec![ + toml::from_str(&format!( + "id = \"sharedid\"\nkind = \"sharedid\"\nsources = [\"pubcid.org\"]\ncollect = \"{sharedid_collect}\"" + )) + .expect("should parse sharedid"), + toml::from_str(&format!( + "id = \"criteo\"\nkind = \"criteo\"\nsources = [\"criteo.com\"]\ncollect = \"{criteo_collect}\"" + )) + .expect("should parse criteo"), + ]; + settings } #[test] fn parses_an_optional_hourly_rate_limit() { let limited: IdentityProviderConfig = toml::from_str( - "id = \"example-provider\"\nkind = \"example-kind\"\nendpoint = \"https://ids.example.com/v1\"\nrate_limit_per_hour = 2", + "id = \"sharedid\"\nkind = \"sharedid\"\nsources = [\"pubcid.org\"]\nrate_limit_per_hour = 2", ) .expect("should parse a rate limit"); - let unset: IdentityProviderConfig = - toml::from_str("id = \"example-provider\"\nkind = \"example-kind\"\nendpoint = \"https://ids.example.com/v1\"") - .expect("should parse without a rate limit"); + let zero = toml::from_str::( + "id = \"sharedid\"\nkind = \"sharedid\"\nsources = [\"pubcid.org\"]\nrate_limit_per_hour = 0", + ); assert_eq!( - limited.rate_limit_per_hour.map(std::num::NonZeroU32::get), + limited.rate_limit_per_hour.map(NonZeroU32::get), Some(2), "should keep the hourly limit" ); + assert!(zero.is_err(), "should reject a zero limit"); + let settings = settings_with_cookie_providers("page", "page"); assert!( - unset.rate_limit_per_hour.is_none(), + settings.ec.identity_providers[0] + .rate_limit_per_hour + .is_none(), "should leave the limit unset by default" ); + } + + #[test] + fn providers_default_to_page_collection() { + let mut settings = settings_with_cookie_providers("page", "page"); + settings.ec.identity_providers = vec![ + toml::from_str("id = \"sharedid\"\nkind = \"sharedid\"\nsources = [\"pubcid.org\"]") + .expect("should parse sharedid"), + ]; + + let page = build_coordinator(&settings).expect("should build page coordinator"); + let after = build_after_response_coordinator(&settings) + .expect("should build after-response coordinator"); + + assert!( + page.is_some(), + "should collect a provider without `collect` on the page" + ); + assert!( + after.is_none(), + "should build no after-response coordinator" + ); + } + + #[test] + fn splits_providers_by_collect_timing() { + let settings = settings_with_cookie_providers("page", "after_response"); + + let page = build_coordinator(&settings) + .expect("should build page coordinator") + .expect("should have a page provider"); + let after = build_after_response_coordinator(&settings) + .expect("should build after-response coordinator") + .expect("should have an after-response provider"); + + assert!(!page.is_empty(), "should hold the page provider"); + assert!(!after.is_empty(), "should hold the after-response provider"); + } + + #[test] + fn all_after_response_builds_no_page_coordinator() { + let settings = settings_with_cookie_providers("after_response", "after_response"); + + let page = build_coordinator(&settings).expect("should build page coordinator"); + let after = build_after_response_coordinator(&settings) + .expect("should build after-response coordinator"); + + assert!(page.is_none(), "should build no page coordinator"); + assert!( + after.is_some(), + "should build the after-response coordinator" + ); + } + + #[test] + fn rejects_a_source_claimed_in_both_phases() { + let mut settings = settings_with_cookie_providers("page", "after_response"); + settings.ec.identity_providers[1] = toml::from_str( + "id = \"sharedid-late\"\nkind = \"sharedid\"\nsources = [\"pubcid.org\"]\ncollect = \"after_response\"", + ) + .expect("should parse second sharedid"); + + let partners = partner_registry(&settings.ec).expect("should build partner registry"); + + let result = validate_enabled_configs(&settings.ec.identity_providers, &partners); + + assert!( + result.is_err(), + "should reject a source claimed by providers in different phases" + ); + } + + #[test] + fn rejects_an_unknown_collect_timing() { toml::from_str::( - "id = \"example-provider\"\nkind = \"example-kind\"\nendpoint = \"https://ids.example.com/v1\"\nrate_limit_per_hour = 0", + "id = \"sharedid\"\nkind = \"sharedid\"\ncollect = \"later\"", ) - .expect_err("should reject a zero limit"); + .expect_err("should reject an unknown `collect` value"); } } diff --git a/crates/trusted-server-identity-providers/src/liveintent.rs b/crates/trusted-server-identity-providers/src/liveintent.rs index bafbb9243..339a5def7 100644 --- a/crates/trusted-server-identity-providers/src/liveintent.rs +++ b/crates/trusted-server-identity-providers/src/liveintent.rs @@ -307,6 +307,7 @@ impl IdentityProvider for LiveIntentProvider { url, headers, body: Bytes::new(), + context: None, }) } @@ -355,6 +356,7 @@ impl IdentityProvider for LiveIntentProvider { atype: Some(LIVEINTENT_ATYPE), ext: uid_ext.map(Cow::Borrowed), expiry, + refresh: None, }], }) }) @@ -552,6 +554,7 @@ mod tests { status: StatusCode::OK, headers: &headers, body: &body, + context: None, }) .expect("should parse response"); let LookupOutcome::Found(eids) = outcome else { @@ -698,6 +701,7 @@ mod tests { status, headers: &headers, body, + context: None, }) .expect("should parse response"); assert!( @@ -721,6 +725,7 @@ mod tests { status, headers: &headers, body, + context: None, }) .expect_err("should reject the response"); assert!( diff --git a/crates/trusted-server-identity-providers/src/liveramp.rs b/crates/trusted-server-identity-providers/src/liveramp.rs index 5b2cb594e..fd30bb820 100644 --- a/crates/trusted-server-identity-providers/src/liveramp.rs +++ b/crates/trusted-server-identity-providers/src/liveramp.rs @@ -7,8 +7,15 @@ //! under its configured storage name (commonly `idl_env`); a value that does //! not decode to JSON is taken as such a bare envelope. //! The envelope is stored unchanged with agent type 3 for -//! [`ENVELOPE_TTL`]. No server call is made: the refresh contract is not -//! confirmed, so the browser keeps refreshing the cookie. +//! [`ENVELOPE_TTL`]. +//! +//! With an `endpoint` (`LiveRamp`'s ATS envelope API, path +//! `/api/identity/v2/envelope`), a visitor without an envelope cookie is looked up server side by Custom ID (`it=15`): the +//! stable part of the EC ID, sent as `iv=:` with +//! the placement ID, consent strings, the publisher `Origin` and the +//! visitor's IP in `X-Forwarded-For`. `LiveRamp` must enable Custom ID for +//! the placement. A `204` reply reports an opt-out. Without an `endpoint` +//! no server call is made. //! //! With the Lockr capability enabled (`[settings.lockr]`), the cookies Lockr //! AIM writes the envelope into (default key `_lr_env`) are read after the @@ -20,9 +27,14 @@ use std::borrow::Cow; use std::time::Duration; +use bytes::Bytes; +use http::header::ORIGIN; +use http::{HeaderMap, HeaderName, HeaderValue, Method, StatusCode}; +use url::Url; + use base64::Engine as _; use base64::engine::general_purpose::{STANDARD, STANDARD_NO_PAD, URL_SAFE, URL_SAFE_NO_PAD}; -use error_stack::Report; +use error_stack::{Report, ResultExt as _}; use serde::Deserialize; use trusted_server_core::ec::eid_types::SourceDomain; use trusted_server_core::ec::provider::{ @@ -33,9 +45,18 @@ use trusted_server_core::ec::provider::{ use crate::common::{ CookieReader, LockrSettings, ReadsLockr, bounded_expiry, cookie_capabilities, cookie_eid, - is_usable_id, percent_decoded, + is_routable_ip, is_usable_id, join_section_ids, percent_decoded, }; +/// `it` value of a Custom ID lookup. +const CUSTOM_ID_TYPE: &str = "15"; +/// `atype` value of an ATS API integration. +const ATS_API: &str = "2"; +/// `ct` value of a TCF v2 consent string. +const TCF_CONSENT: &str = "4"; +/// `ct` value of a US Privacy consent string. +const US_PRIVACY_CONSENT: &str = "3"; + /// How long a stored envelope is kept. pub const ENVELOPE_TTL: Duration = Duration::from_secs(15 * 24 * 60 * 60); @@ -52,6 +73,13 @@ pub struct LiveRampSettings { /// The Lockr capability; see [`LockrSettings`]. #[serde(default)] pub lockr: LockrSettings, + /// `LiveRamp` Placement ID (`pid`); required with an `endpoint`. + #[serde(default)] + pub placement_id: Option, + /// `LiveRamp`-assigned namespace of the Custom ID (the `ANA_ID` part of + /// `iv`); required with an `endpoint`. + #[serde(default)] + pub custom_id_namespace: Option, } impl LiveRampSettings { @@ -66,6 +94,16 @@ pub struct LiveRampProvider { capabilities: ProviderCapabilities, source: SourceDomain, cookies: CookieReader, + /// The envelope lookup; `None` disables server calls. + lookup: Option, +} + +/// A configured Custom ID envelope lookup. +#[derive(Debug)] +struct EnvelopeLookup { + endpoint: Url, + placement_id: u64, + custom_id_namespace: String, } impl ReadsLockr for LiveRamp { @@ -88,10 +126,31 @@ impl ProviderKind for LiveRamp { .attach("a liveramp provider needs at least one cookie name")); } let (capabilities, source) = cookie_capabilities(common, Self::KIND)?; + let lookup = match ( + common.endpoint, + settings.placement_id, + settings.custom_id_namespace, + ) { + (None, _, _) => None, + (Some(endpoint), Some(placement_id), Some(custom_id_namespace)) + if !custom_id_namespace.trim().is_empty() => + { + Some(EnvelopeLookup { + endpoint: endpoint.clone(), + placement_id, + custom_id_namespace, + }) + } + (Some(_), _, _) => { + return Err(Report::new(ProviderConfigError::Invalid) + .attach("a liveramp endpoint needs placement_id and custom_id_namespace")); + } + }; Ok(LiveRampProvider { capabilities, source, cookies, + lookup, }) } } @@ -137,16 +196,89 @@ impl IdentityProvider for LiveRampProvider { fn build_request( &self, - _lookup: &LookupInput<'_>, + lookup: &LookupInput<'_>, ) -> Result> { - Err(Report::new(ProviderError::Request).attach("liveramp makes no lookup")) + let Some(envelope_lookup) = &self.lookup else { + return Err(Report::new(ProviderError::Request) + .attach("liveramp looks up only with an endpoint")); + }; + // LiveRamp requires the visitor's IP on server-to-server calls. + let ip = lookup + .client + .ip + .filter(|ip| is_routable_ip(*ip)) + .ok_or_else(|| { + Report::new(ProviderError::Request).attach("liveramp needs a routable client IP") + })?; + let stable_id = lookup.ec_id.split('.').next().unwrap_or(lookup.ec_id); + let consent = lookup.consent; + let mut url = envelope_lookup.endpoint.clone(); + { + let mut query = url.query_pairs_mut(); + query + .append_pair("pid", &envelope_lookup.placement_id.to_string()) + .append_pair("it", CUSTOM_ID_TYPE) + .append_pair( + "iv", + &format!("{}:{stable_id}", envelope_lookup.custom_id_namespace), + ) + .append_pair("atype", ATS_API); + if let Some(tc_string) = &consent.raw_tc_string { + query + .append_pair("ct", TCF_CONSENT) + .append_pair("cv", tc_string); + } else if let Some(us_privacy) = &consent.raw_us_privacy { + query + .append_pair("ct", US_PRIVACY_CONSENT) + .append_pair("cv", us_privacy); + } + if let Some(gpp) = &consent.raw_gpp_string { + query.append_pair("gpp", gpp); + } + if let Some(section_ids) = &consent.gpp_section_ids { + query.append_pair("gpp_sid", &join_section_ids(section_ids)); + } + } + let mut headers = HeaderMap::new(); + headers.insert( + ORIGIN, + HeaderValue::from_str(&format!("https://{}", lookup.publisher_domain)) + .change_context(ProviderError::Request)?, + ); + headers.insert( + HeaderName::from_static("x-forwarded-for"), + HeaderValue::from_str(&ip.to_string()).change_context(ProviderError::Request)?, + ); + Ok(ProviderRequest { + method: Method::GET, + url, + headers, + body: Bytes::new(), + context: None, + }) } fn parse_response<'body>( &self, - _response: ProviderResponse<'body>, + response: ProviderResponse<'body>, ) -> Result, Report> { - Err(Report::new(ProviderError::InvalidResponse).attach("liveramp makes no lookup")) + if response.status == StatusCode::NO_CONTENT { + return Ok(LookupOutcome::OptedOut); + } + if !response.status.is_success() { + return Err(Report::new(ProviderError::InvalidResponse) + .attach(format!("liveramp returned status {}", response.status))); + } + let reply: EnvelopeCookie = + serde_json::from_slice(response.body).change_context(ProviderError::InvalidResponse)?; + if !is_usable_id(&reply.envelope) { + return Ok(LookupOutcome::NoMatch); + } + Ok(LookupOutcome::Found(vec![cookie_eid( + &self.source, + Cow::Owned(reply.envelope), + bounded_expiry(Some(ENVELOPE_TTL)), + )])) } fn read_request_cookies<'cookie>( @@ -393,7 +525,7 @@ mod tests { } #[test] - fn makes_no_lookup() { + fn makes_no_lookup_without_an_endpoint() { let provider = provider(&json!({})); let consent = ConsentContext::default(); let lookup = LookupInput { @@ -415,4 +547,217 @@ mod tests { "should build no request" ); } + + fn lookup_provider() -> Box { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "liveramp", + "kind": "liveramp", + "endpoint": "https://envelope.example.com/api/identity/v2/envelope", + "sources": ["liveramp.example.com"], + "settings": {"placement_id": 14, "custom_id_namespace": "ana-example"}, + })) + .expect("should parse provider config"); + build_kind::(&config, &ProviderSecrets::default()).expect("should build provider") + } + + fn input<'a>(consent: &'a ConsentContext, ip: Option<&str>) -> LookupInput<'a> { + LookupInput { + ec_id: "abc123.suffix", + consent, + client: ClientSignals { + ip: ip.map(|ip| ip.parse().expect("should parse ip")), + user_agent: None, + }, + publisher_domain: "www.example.com", + due_sources: &[], + current: OwnedEntries::new(None, &[]), + cookies: RequestCookies::default(), + hashed_email: None, + } + } + + fn query(request: &ProviderRequest) -> Vec<(String, String)> { + request + .url + .query_pairs() + .map(|(key, value)| (key.into_owned(), value.into_owned())) + .collect() + } + + fn pairs(expected: &[(&str, &str)]) -> Vec<(String, String)> { + expected + .iter() + .map(|(key, value)| ((*key).to_owned(), (*value).to_owned())) + .collect() + } + + #[test] + fn looks_up_an_envelope_by_custom_id() { + let provider = lookup_provider(); + let consent = ConsentContext { + raw_tc_string: Some("CP_example".to_owned()), + raw_gpp_string: Some("DBA_example".to_owned()), + gpp_section_ids: Some(vec![2, 7]), + ..ConsentContext::default() + }; + + let request = provider + .build_request(&input(&consent, Some("11.22.33.44"))) + .expect("should build request"); + + assert_eq!(request.method, Method::GET, "should get"); + assert_eq!( + request.url.path(), + "/api/identity/v2/envelope", + "should call the configured endpoint" + ); + assert_eq!( + query(&request), + pairs(&[ + ("pid", "14"), + ("it", "15"), + ("iv", "ana-example:abc123"), + ("atype", "2"), + ("ct", "4"), + ("cv", "CP_example"), + ("gpp", "DBA_example"), + ("gpp_sid", "2,7"), + ]), + "should send the stable EC ID as the Custom ID with consent" + ); + assert_eq!( + request.headers.get(ORIGIN).map(HeaderValue::as_bytes), + Some(&b"https://www.example.com"[..]), + "should send the publisher origin" + ); + assert_eq!( + request + .headers + .get("x-forwarded-for") + .map(HeaderValue::as_bytes), + Some(&b"11.22.33.44"[..]), + "should forward the visitor IP" + ); + } + + #[test] + fn sends_us_privacy_when_there_is_no_tcf_string() { + let provider = lookup_provider(); + let consent = ConsentContext { + raw_us_privacy: Some("1YNN".to_owned()), + ..ConsentContext::default() + }; + + let request = provider + .build_request(&input(&consent, Some("11.22.33.44"))) + .expect("should build request"); + + let query = query(&request); + assert!( + query.contains(&("ct".to_owned(), "3".to_owned())) + && query.contains(&("cv".to_owned(), "1YNN".to_owned())), + "should send the US Privacy string" + ); + } + + #[test] + fn needs_a_routable_client_ip() { + let provider = lookup_provider(); + let consent = ConsentContext::default(); + + for ip in [None, Some("10.0.0.1")] { + assert!( + provider.build_request(&input(&consent, ip)).is_err(), + "should not look up without a routable IP" + ); + } + } + + fn reply(status: StatusCode, body: &[u8]) -> Result, Report> { + static HEADERS: std::sync::LazyLock = std::sync::LazyLock::new(HeaderMap::new); + lookup_provider().parse_response(ProviderResponse { + status, + headers: &HEADERS, + body, + context: None, + }) + } + + #[test] + fn stores_the_returned_envelope() { + let body = json!({"envelope": "AhIdEnvelope"}).to_string(); + + let outcome = reply(StatusCode::OK, body.as_bytes()).expect("should parse reply"); + + let LookupOutcome::Found(eids) = outcome else { + panic!("should find an envelope"); + }; + assert_eq!( + eids[0].source, "liveramp.example.com", + "should use the source" + ); + assert_eq!( + eids[0].uids[0].id, "AhIdEnvelope", + "should store the envelope" + ); + assert_eq!(eids[0].uids[0].atype, Some(3), "should be person-based"); + assert!( + matches!(eids[0].uids[0].expiry, ProviderExpiry::After(ttl) if ttl == ENVELOPE_TTL), + "should keep the envelope for the envelope TTL" + ); + } + + #[test] + fn maps_no_content_to_an_opt_out_and_a_blank_envelope_to_no_match() { + let blank = json!({"envelope": ""}).to_string(); + + assert!( + matches!( + reply(StatusCode::NO_CONTENT, b""), + Ok(LookupOutcome::OptedOut) + ), + "should report an opt-out" + ); + assert!( + matches!( + reply(StatusCode::OK, blank.as_bytes()), + Ok(LookupOutcome::NoMatch) + ), + "should report no match" + ); + } + + #[test] + fn rejects_an_error_status_and_a_malformed_body() { + for (status, body) in [ + (StatusCode::UNAVAILABLE_FOR_LEGAL_REASONS, &b"{}"[..]), + (StatusCode::UNAUTHORIZED, &b""[..]), + (StatusCode::OK, &b"not json"[..]), + ] { + assert!(reply(status, body).is_err(), "should reject {status}"); + } + } + + #[test] + fn an_endpoint_needs_a_placement_and_namespace() { + for settings in [ + json!({"placement_id": 14}), + json!({"custom_id_namespace": "ana-example"}), + json!({"placement_id": 14, "custom_id_namespace": " "}), + ] { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "liveramp", + "kind": "liveramp", + "endpoint": "https://envelope.example.com/api/identity/v2/envelope", + "sources": ["liveramp.example.com"], + "settings": settings, + })) + .expect("should parse provider config"); + + assert!( + build_kind::(&config, &ProviderSecrets::default()).is_err(), + "should reject {settings}" + ); + } + } } diff --git a/crates/trusted-server-identity-providers/src/sharedid.rs b/crates/trusted-server-identity-providers/src/sharedid.rs index 0835e6ecb..b9c08dd3f 100644 --- a/crates/trusted-server-identity-providers/src/sharedid.rs +++ b/crates/trusted-server-identity-providers/src/sharedid.rs @@ -6,6 +6,12 @@ //! (agent type 1) EID. The opt-out cookie `_pubcid_optout` reports an //! opt-out, which removes a stored `SharedID`. //! No server call is made. +//! +//! `SharedID` is a publisher first-party ID with no vendor server, so when +//! the cookies hold none and no live `SharedID` is stored, Trusted Server +//! generates one itself: a random UUID v4, stored in the row only. No cookie +//! is set, so Prebid's own cookie is never touched; when Prebid later writes +//! one, the cookie's ID replaces the generated one. use std::borrow::Cow; @@ -14,11 +20,14 @@ use serde::Deserialize; use trusted_server_core::ec::eid_types::SourceDomain; use trusted_server_core::ec::provider::{ CookieRead, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, - ProviderConfigError, ProviderError, ProviderKind, ProviderRequest, ProviderResponse, - RequestCookies, + ProviderConfigError, ProviderEid, ProviderError, ProviderKind, ProviderRequest, + ProviderResponse, RequestCookies, }; +use uuid::Uuid; -use crate::common::{CookieReader, bounded_expiry, cookie_capabilities, device_eid, is_usable_id}; +use crate::common::{ + CookieReader, MAX_TTL, bounded_expiry, cookie_capabilities, device_eid, is_usable_id, +}; /// Cookie Prebid sets when the user opts out of `SharedID`. const OPT_OUT_COOKIE: &str = "_pubcid_optout"; @@ -109,12 +118,22 @@ impl IdentityProvider for SharedIdProvider { bounded_expiry(None), )]) } + + fn generate(&self, _lookup: &LookupInput<'_>) -> Option>> { + Some(vec![device_eid( + &self.source, + Cow::Owned(Uuid::new_v4().to_string()), + // A generated ID has no cookie to renew it, so it keeps the + // longest lifetime rather than changing every day. + bounded_expiry(Some(MAX_TTL)), + )]) + } } #[cfg(test)] mod tests { use serde_json::json; - use trusted_server_core::ec::provider::build_kind; + use trusted_server_core::ec::provider::{ProviderExpiry, build_kind}; use trusted_server_core::settings::{IdentityProviderConfig, ProviderSecrets}; use super::*; @@ -171,4 +190,42 @@ mod tests { assert!(read(header).is_none(), "should ignore {header}"); } } + + #[test] + fn generates_a_random_uuid_when_none_is_held() { + let consent = trusted_server_core::consent::ConsentContext::default(); + let lookup = LookupInput { + ec_id: "ec-example", + consent: &consent, + client: trusted_server_core::ec::provider::ClientSignals { + ip: None, + user_agent: None, + }, + publisher_domain: "www.example.com", + due_sources: &[], + current: trusted_server_core::ec::provider::OwnedEntries::new(None, &[]), + cookies: RequestCookies::default(), + hashed_email: None, + }; + let provider = provider(); + + let first = provider.generate(&lookup).expect("should generate an ID"); + let second = provider.generate(&lookup).expect("should generate an ID"); + + let uid = &first[0].uids[0]; + assert_eq!( + first[0].source, "sharedid.example.com", + "should use the source" + ); + assert_eq!(uid.atype, Some(1), "should be device-based"); + assert!( + Uuid::parse_str(&uid.id).is_ok_and(|id| id.get_version_num() == 4), + "should be a UUID v4" + ); + assert_ne!(uid.id, second[0].uids[0].id, "should be random"); + assert!( + matches!(uid.expiry, ProviderExpiry::After(ttl) if ttl == MAX_TTL), + "should keep a generated ID for the longest TTL" + ); + } } diff --git a/crates/trusted-server-identity-providers/src/uid2.rs b/crates/trusted-server-identity-providers/src/uid2.rs index 7ff1cd6e6..f025113f0 100644 --- a/crates/trusted-server-identity-providers/src/uid2.rs +++ b/crates/trusted-server-identity-providers/src/uid2.rs @@ -1,12 +1,21 @@ //! UID2 identity provider: reads the UID2 advertising token from the -//! browser's own first-party cookies. +//! browser's own first-party cookies, and refreshes it server side when an +//! `endpoint` is configured. //! //! The UID2 SDK stores its identity in the `__uid_2` cookie as -//! percent-encoded JSON. Only `advertising_token` and `identity_expires` +//! percent-encoded JSON. `advertising_token` and `identity_expires` //! (milliseconds) are read: the token is stored with agent type 3 until //! `identity_expires`, and the coordinator drops it once that time has -//! passed. The refresh token and response key are never read, so they are -//! never stored, and no refresh is made server side. +//! passed. +//! +//! With an `endpoint` (UID2's `POST /v2/token/refresh`), the cookie's +//! `refresh_token` and `refresh_response_key` are also kept with the token as +//! private refresh state, and the source becomes due at `refresh_from`. The +//! refresh posts the refresh token, which needs no API key and no email, and +//! decrypts the reply with the response key (AES-256-GCM, a 12-byte IV +//! first). A refresh replying `optout` removes the stored token. No token is +//! ever generated: that needs an email or phone number, which is out of +//! scope. Without an `endpoint` the refresh state is not stored. //! //! When `__uid_2` yields nothing, the cookies in `cookie_names` (default //! Prebid's `__uid2_advertising_token`) are read. Prebid's UID2 module @@ -28,14 +37,21 @@ use std::borrow::Cow; +use aes_gcm::aead::{Aead as _, KeyInit as _}; +use aes_gcm::{Aes256Gcm, Nonce}; +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; +use bytes::Bytes; use error_stack::Report; -use serde::Deserialize; +use http::header::{CONTENT_TYPE, HeaderMap, HeaderValue}; +use serde::{Deserialize, Serialize}; use trusted_server_core::ec::eid_types::{SourceDomain, UnixSeconds}; use trusted_server_core::ec::provider::{ CookieRead, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, - ProviderConfigError, ProviderError, ProviderExpiry, ProviderKind, ProviderRequest, - ProviderResponse, RequestCookies, + ProviderConfigError, ProviderEid, ProviderError, ProviderExpiry, ProviderKind, ProviderRefresh, + ProviderRequest, ProviderResponse, RequestCookies, }; +use url::Url; use crate::common::{ CookieReader, LockrSettings, ReadsLockr, bounded_expiry, cookie_capabilities, cookie_eid, @@ -78,6 +94,8 @@ pub struct Uid2Provider { source: SourceDomain, identity_cookie_name: String, cookies: CookieReader, + /// UID2's token refresh endpoint; `None` disables server refresh. + refresh_endpoint: Option, } impl ReadsLockr for Uid2 { @@ -103,26 +121,99 @@ impl ProviderKind for Uid2 { settings.cookie_names, settings.lockr, )?, + refresh_endpoint: common.endpoint.cloned(), }) } } -/// The fields of the `__uid_2` cookie this provider reads. +/// A UID2 identity: the `__uid_2` cookie and the `body` of a decrypted +/// refresh reply share these fields. #[derive(Deserialize)] -struct IdentityCookie { +struct Identity { advertising_token: String, identity_expires: u64, + #[serde(default)] + refresh_token: Option, + #[serde(default)] + refresh_response_key: Option, + #[serde(default)] + refresh_from: Option, +} + +/// The refresh state kept with a stored token. +#[derive(Serialize, Deserialize)] +struct RefreshState { + /// The refresh token posted to the refresh endpoint. + token: String, + /// Base64 AES-256 key that decrypts the refresh reply. + key: String, +} + +/// A usable token with its expiry and, when present, its refresh state. +struct Token { + token: String, + expiry: ProviderExpiry, + refresh: Option<(RefreshState, ProviderExpiry)>, } -/// Decodes one `__uid_2` value into the token and its expiry. -fn identity(value: &str) -> Option<(String, ProviderExpiry)> { +impl Identity { + /// The token, with refresh state when the identity carries all of it. + fn into_token(self) -> Option { + if !is_usable_id(&self.advertising_token) { + return None; + } + let refresh = match ( + self.refresh_token, + self.refresh_response_key, + self.refresh_from, + ) { + (Some(token), Some(key), Some(from)) if !token.is_empty() && !key.is_empty() => Some(( + RefreshState { token, key }, + ProviderExpiry::At(UnixSeconds::new(from / 1000)), + )), + _ => None, + }; + Some(Token { + token: self.advertising_token, + expiry: ProviderExpiry::At(UnixSeconds::new(self.identity_expires / 1000)), + refresh, + }) + } +} + +/// Decodes one `__uid_2` value. +fn identity(value: &str) -> Option { let decoded = percent_decoded(value); - let cookie: IdentityCookie = serde_json::from_str(&decoded).ok()?; - if !is_usable_id(&cookie.advertising_token) { + serde_json::from_str::(&decoded) + .ok()? + .into_token() +} + +/// A decrypted refresh reply. +#[derive(Deserialize)] +struct RefreshReply { + status: String, + #[serde(default)] + body: Option, +} + +/// Status of a refresh reply for a user who opted out. +const REFRESH_OPT_OUT: &str = "optout"; + +/// Length of the AES-GCM IV that starts a refresh reply. +const IV_BYTES: usize = 12; + +/// Decrypts a base64 refresh reply with the base64 `key`. +fn decrypt_reply(body: &[u8], key: &str) -> Option> { + let key = BASE64.decode(key).ok()?; + let cipher = Aes256Gcm::new_from_slice(&key).ok()?; + let envelope = BASE64.decode(body.trim_ascii()).ok()?; + if envelope.len() <= IV_BYTES { return None; } - let expires = UnixSeconds::new(cookie.identity_expires / 1000); - Some((cookie.advertising_token, ProviderExpiry::At(expires))) + let (iv, ciphertext) = envelope.split_at(IV_BYTES); + let nonce = Nonce::try_from(iv).ok()?; + cipher.decrypt(&nonce, ciphertext).ok() } /// `latestToken` value Prebid stores after an opt-out. @@ -187,16 +278,57 @@ impl IdentityProvider for Uid2Provider { fn build_request( &self, - _lookup: &LookupInput<'_>, + lookup: &LookupInput<'_>, ) -> Result> { - Err(Report::new(ProviderError::Request).attach("uid2 makes no lookup")) + let Some(endpoint) = &self.refresh_endpoint else { + return Err(Report::new(ProviderError::Request) + .attach("uid2 refreshes only with an endpoint and never generates tokens")); + }; + let state = lookup + .current + .iter() + .flat_map(|(_, eid)| eid.uids()) + .find_map(|uid| uid.refresh()) + .and_then(|refresh| serde_json::from_str::(&refresh.state).ok()) + .ok_or_else(|| { + Report::new(ProviderError::Request).attach("no stored uid2 refresh token") + })?; + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_TYPE, HeaderValue::from_static("text/plain")); + Ok(ProviderRequest { + method: http::Method::POST, + url: endpoint.clone(), + headers, + body: Bytes::from(state.token), + context: Some(state.key), + }) } fn parse_response<'body>( &self, - _response: ProviderResponse<'body>, + response: ProviderResponse<'body>, ) -> Result, Report> { - Err(Report::new(ProviderError::InvalidResponse).attach("uid2 makes no lookup")) + let key = response.context.ok_or_else(|| { + Report::new(ProviderError::InvalidResponse).attach("no uid2 response key") + })?; + let plaintext = decrypt_reply(response.body, key).ok_or_else(|| { + Report::new(ProviderError::InvalidResponse).attach("uid2 reply not decrypted") + })?; + let reply: RefreshReply = serde_json::from_slice(&plaintext).map_err(|error| { + Report::new(ProviderError::InvalidResponse).attach(format!("uid2 reply: {error}")) + })?; + if reply.status == REFRESH_OPT_OUT { + return Ok(LookupOutcome::OptedOut); + } + let token = reply + .body + .filter(|_| reply.status == "success") + .and_then(Identity::into_token) + .ok_or_else(|| { + Report::new(ProviderError::InvalidResponse) + .attach(format!("uid2 refresh status {}", reply.status)) + })?; + Ok(LookupOutcome::Found(vec![self.eid(token)])) } fn read_request_cookies<'cookie>( @@ -215,14 +347,38 @@ impl IdentityProvider for Uid2Provider { .or_else(|| { self.cookies .read(cookies, |value| match token_cookie(value)? { - TokenCookie::Token(token, expiry) => Some((token, expiry)), + TokenCookie::Token(token, expiry) => Some(Token { + token, + expiry, + refresh: None, + }), TokenCookie::OptedOut => None, }) }); - let Some((token, expiry)) = token else { + let Some(token) = token else { return CookieRead::Absent; }; - CookieRead::Found(vec![cookie_eid(&self.source, Cow::Owned(token), expiry)]) + CookieRead::Found(vec![self.eid(token)]) + } +} + +impl Uid2Provider { + /// Builds the entry for `token`, keeping its refresh state only when + /// server refresh is enabled. + fn eid<'any>(&self, token: Token) -> ProviderEid<'any> { + let mut eid = cookie_eid(&self.source, Cow::Owned(token.token), token.expiry); + if self.refresh_endpoint.is_some() + && let Some((state, from)) = token.refresh + && let Ok(state) = serde_json::to_string(&state) + { + for uid in &mut eid.uids { + uid.refresh = Some(ProviderRefresh { + state: Cow::Owned(state.clone()), + from, + }); + } + } + eid } } @@ -232,6 +388,11 @@ mod tests { use trusted_server_core::ec::provider::build_kind; use trusted_server_core::settings::{IdentityProviderConfig, ProviderSecrets}; + use trusted_server_core::consent::ConsentContext; + use trusted_server_core::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, UidValue}; + use trusted_server_core::ec::kv_types::{KvEid, KvEntry, KvPartnerId, KvRefresh, KvUid}; + use trusted_server_core::ec::provider::{ClientSignals, OwnedEntries}; + use super::*; fn provider(settings: &serde_json::Value) -> Box { @@ -432,4 +593,256 @@ mod tests { ); } } + + const REFRESH_ENDPOINT: &str = "https://uid2.example.com/v2/token/refresh"; + + fn refreshing_provider() -> Box { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "uid2", + "kind": "uid2", + "sources": ["uid2.example.com"], + "endpoint": REFRESH_ENDPOINT, + "allowed_domains": ["uid2.example.com"], + })) + .expect("should parse provider config"); + build_kind::(&config, &ProviderSecrets::default()).expect("should build provider") + } + + fn sdk_identity_header() -> String { + identity_cookie(&json!({ + "advertising_token": "AgAAAAexampletoken", + "refresh_token": "example-refresh-token", + "identity_expires": 1_700_000_000_000_u64, + "refresh_from": 1_699_990_000_000_u64, + "refresh_expires": 1_702_000_000_000_u64, + "refresh_response_key": BASE64.encode([7_u8; 32]), + })) + } + + fn refresh_of( + provider: &dyn IdentityProvider, + header: &str, + ) -> Option<(String, ProviderExpiry)> { + let eids = provider + .read_request_cookies(RequestCookies::new(header)) + .into_found() + .expect("should read the token"); + let refresh = eids[0].uids[0].refresh.as_ref()?; + Some((refresh.state.to_string(), refresh.from)) + } + + #[test] + fn keeps_the_refresh_state_when_refresh_is_enabled() { + let (state, from) = refresh_of(&*refreshing_provider(), &sdk_identity_header()) + .expect("should keep the refresh state"); + + let state: RefreshState = serde_json::from_str(&state).expect("should parse the state"); + assert_eq!( + state.token, "example-refresh-token", + "should keep the refresh token" + ); + assert!( + matches!(from, ProviderExpiry::At(at) if at.get() == 1_699_990_000), + "should ask for a refresh at refresh_from" + ); + } + + #[test] + fn stores_no_refresh_state_without_an_endpoint() { + assert!( + refresh_of(&*provider(&json!({})), &sdk_identity_header()).is_none(), + "should not keep the refresh token when refresh is disabled" + ); + } + + fn stored_entry() -> KvEntry { + let state = serde_json::to_string(&RefreshState { + token: "example-refresh-token".to_owned(), + key: BASE64.encode([7_u8; 32]), + }) + .expect("should serialize the state"); + let uid = KvUid::new( + UidValue::try_from("AgAAAAexampletoken".to_owned()).expect("should build uid"), + AgentType::try_from(3).expect("should build atype"), + None, + Some(ProviderId::new("uid2")), + Expiry::At(UnixSeconds::new(1_700_000_000)), + ) + .with_refresh(KvRefresh { + state, + from: UnixSeconds::new(1_699_990_000), + }); + let record = KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![uid])], + UnixSeconds::new(1_699_995_000), + ) + .expect("should keep the live uid"); + let mut entry = KvEntry::tombstone(1); + entry.consent.ok = true; + entry.ids.insert("uid2.example.com".to_owned(), record); + entry + } + + #[test] + fn refresh_posts_the_stored_refresh_token() { + let entry = stored_entry(); + let sources = [SourceDomain::try_from("uid2.example.com").expect("should build source")]; + let consent = ConsentContext::default(); + let lookup = LookupInput { + ec_id: "ec-example", + consent: &consent, + client: ClientSignals { + ip: None, + user_agent: None, + }, + publisher_domain: "www.example.com", + due_sources: &sources, + current: OwnedEntries::new(Some(&entry), &sources), + cookies: RequestCookies::default(), + hashed_email: None, + }; + + let request = refreshing_provider() + .build_request(&lookup) + .expect("should build the refresh request"); + + assert_eq!(request.method, http::Method::POST, "should post"); + assert_eq!( + request.url.as_str(), + REFRESH_ENDPOINT, + "should call the endpoint" + ); + assert_eq!( + request.body.as_ref(), + b"example-refresh-token", + "should post the refresh token only" + ); + assert_eq!( + request.context, + Some(BASE64.encode([7_u8; 32])), + "should keep the response key for decryption" + ); + } + + #[test] + fn makes_no_request_without_an_endpoint() { + let entry = stored_entry(); + let sources = [SourceDomain::try_from("uid2.example.com").expect("should build source")]; + let consent = ConsentContext::default(); + let lookup = LookupInput { + ec_id: "ec-example", + consent: &consent, + client: ClientSignals { + ip: None, + user_agent: None, + }, + publisher_domain: "www.example.com", + due_sources: &sources, + current: OwnedEntries::new(Some(&entry), &sources), + cookies: RequestCookies::default(), + hashed_email: None, + }; + + assert!( + provider(&json!({})).build_request(&lookup).is_err(), + "should never generate a token" + ); + } + + fn encrypted_reply(reply: &serde_json::Value) -> String { + let cipher = Aes256Gcm::new_from_slice(&[7_u8; 32]).expect("should build cipher"); + let iv = [1_u8; IV_BYTES]; + let ciphertext = cipher + .encrypt( + &Nonce::try_from(&iv[..]).expect("should build nonce"), + reply.to_string().as_bytes(), + ) + .expect("should encrypt"); + BASE64.encode([&iv[..], &ciphertext].concat()) + } + + fn reply<'a>(body: &'a str, key: &'a str, headers: &'a HeaderMap) -> ProviderResponse<'a> { + ProviderResponse { + status: http::StatusCode::OK, + headers, + body: body.as_bytes(), + context: Some(key), + } + } + + #[test] + fn decrypts_a_refreshed_identity() { + let body = encrypted_reply(&json!({ + "status": "success", + "body": { + "advertising_token": "AgAAAArefreshedtoken", + "refresh_token": "next-refresh-token", + "identity_expires": 1_700_100_000_000_u64, + "refresh_from": 1_700_090_000_000_u64, + "refresh_expires": 1_700_200_000_000_u64, + "refresh_response_key": BASE64.encode([9_u8; 32]), + }, + })); + + let (key, headers) = (BASE64.encode([7_u8; 32]), HeaderMap::new()); + + let outcome = refreshing_provider() + .parse_response(reply(&body, &key, &headers)) + .expect("should parse the reply"); + + let LookupOutcome::Found(eids) = outcome else { + panic!("should find the refreshed token"); + }; + let uid = &eids[0].uids[0]; + assert_eq!(uid.id, "AgAAAArefreshedtoken", "should store the new token"); + assert!( + matches!(uid.expiry, ProviderExpiry::At(at) if at.get() == 1_700_100_000), + "should expire at the new identity_expires" + ); + let refresh = uid + .refresh + .as_ref() + .expect("should keep the next refresh state"); + let state: RefreshState = + serde_json::from_str(&refresh.state).expect("should parse the state"); + assert_eq!( + state.token, "next-refresh-token", + "should keep the next refresh token" + ); + assert_eq!( + state.key, + BASE64.encode([9_u8; 32]), + "should keep the next key" + ); + } + + #[test] + fn reports_an_opt_out_from_the_refresh() { + let body = encrypted_reply(&json!({ "status": "optout" })); + + let (key, headers) = (BASE64.encode([7_u8; 32]), HeaderMap::new()); + + let outcome = refreshing_provider() + .parse_response(reply(&body, &key, &headers)) + .expect("should parse the reply"); + + assert!( + matches!(outcome, LookupOutcome::OptedOut), + "should remove the stored token on opt-out" + ); + } + + #[test] + fn rejects_a_reply_under_another_key() { + let body = encrypted_reply(&json!({ "status": "optout" })); + + let (key, headers) = (BASE64.encode([8_u8; 32]), HeaderMap::new()); + + assert!( + refreshing_provider() + .parse_response(reply(&body, &key, &headers)) + .is_err(), + "should reject a reply that does not decrypt" + ); + } } diff --git a/docs/guide/api-reference.md b/docs/guide/api-reference.md index 83d580692..09fd5b000 100644 --- a/docs/guide/api-reference.md +++ b/docs/guide/api-reference.md @@ -884,6 +884,7 @@ This lookup is implemented only by the Fastly adapter because the identity graph - `ec_id`, `store`, and `generation` identify the raw KV lookup. - `entry` preserves the stored JSON shape, including unknown and legacy fields. Derived `created_iso` and `consent.updated_iso` fields are added only when absent. - `metadata` preserves the stored metadata JSON shape. +- Provider refresh state is private: every `refresh.state` in `entry` is shown as `"[redacted]"`, while `refresh.from` stays visible. - `tombstone` reports whether consent has been withdrawn. It is absent when the entry body cannot be parsed as JSON or deserialized as the typed EC schema. - `auction.eids` previews the partner EIDs the stored record can contribute; `auction.skipped` explains filtered IDs. - `entry_error`, `metadata_error`, and `raw_body` keep malformed or schema-incompatible records inspectable. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 29bdb7d9d..c9235fe47 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -904,8 +904,8 @@ TRUSTED_SERVER__EC__EC_STORE=ec_identity_store ### Identity providers -`[[ec.identity_providers]]` entries look up a visitor's IDs from a provider's -server on publisher page navigations. Lookups start before the origin fetch, +`[[ec.identity_providers]]` entries collect a visitor's IDs from the vendor's +first-party cookies or a provider's server on publisher page navigations. Lookups start before the origin fetch, share the `identity_budget_ms` budget, and are collected before the response is finalized. Results go into the navigation's single EC row write. A lookup that times out or fails leaves the row unchanged. @@ -940,19 +940,25 @@ When a provider reports an opt-out, its sources are cleared and the row remembers the opt-out for 30 days. The provider is not called for that EC again until the 30 days pass. -| Field | Type | Required | Description | -| --------------------- | ------- | -------- | ----------------------------------------------------------------------- | -| `id` | String | Yes | Stable provider ID used in logs, metrics and KV ownership | -| `kind` | String | Yes | Provider implementation; must be one this build includes | -| `enabled` | Boolean | No | Whether the provider runs (default `true`) | -| `endpoint` | String | Yes | HTTPS lookup URL; its host must be in `allowed_domains` | -| `allowed_domains` | Array | Yes | Hosts the provider may call | -| `sources` | Array | Yes | Source domains the provider owns; each needs an `[[ec.partners]]` entry | -| `tcf_vendor_id` | Integer | No | TCF vendor ID that must have vendor consent | -| `collect` | String | No | `"page"` (default) or `"after_response"`; see below | -| `rate_limit_per_hour` | Integer | No | Most lookups per EC hash per hour; unset means unlimited. See below | -| `secrets.token` | String | No | Key in `trusted_server_secrets` for the provider's API token | -| `settings` | Table | No | Provider-specific settings, parsed by the provider kind | +Some kinds keep private refresh state with a UID, such as a vendor refresh +token, and refresh it server side once it is due, while the UID is still live. +Refresh state is never served, mirrored, forwarded or shown: the admin view +shows it as `[redacted]`. Kinds without a vendor server may generate an ID into +the EC row when the cookies hold none; no cookie is set. + +| Field | Type | Required | Description | +| --------------------- | ------- | -------- | ------------------------------------------------------------------------------------- | +| `id` | String | Yes | Stable provider ID used in logs, metrics and KV ownership | +| `kind` | String | Yes | Provider implementation; must be one this build includes | +| `enabled` | Boolean | No | Whether the provider runs (default `true`) | +| `endpoint` | String | No | HTTPS lookup URL, for kinds that make a lookup; its host must be in `allowed_domains` | +| `allowed_domains` | Array | No | Hosts the provider may call; needed with `endpoint` | +| `sources` | Array | Yes | Source domains the provider owns; each needs an `[[ec.partners]]` entry | +| `tcf_vendor_id` | Integer | No | TCF vendor ID that must have vendor consent | +| `collect` | String | No | `"page"` (default) or `"after_response"`; see below | +| `rate_limit_per_hour` | Integer | No | Most lookups per EC hash per hour; unset means unlimited. See below | +| `secrets.token` | String | No | Key in `trusted_server_secrets` for the provider's API token | +| `settings` | Table | No | Provider-specific settings, parsed by the provider kind | **Example**: diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-connectid.md b/docs/superpowers/plans/2026-11-02-identity-provider-connectid.md index 7660c3eaa..dc8e4a37f 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-connectid.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-connectid.md @@ -2,6 +2,8 @@ > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. +**Status:** Implemented; checkboxes are not updated. + **Goal:** Store Yahoo ConnectID from Yahoo's own first-party cookie, as an `[[ec.identity_providers]]` entry with `kind = "connectid"`. No login and no server call. **Architecture:** `crates/trusted-server-identity-providers/src/connectid.rs` holds `ConnectId` (a `ProviderKind`), `ConnectIdSettings` and `ConnectIdProvider`. The provider implements only `IdentityProvider::read_request_cookies`, using the cookie helpers in `common.rs`; `build_request` always fails, so no lookup is sent. diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-criteo.md b/docs/superpowers/plans/2026-11-02-identity-provider-criteo.md index c33edb060..d0f07847a 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-criteo.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-criteo.md @@ -2,6 +2,8 @@ > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. +**Status:** Implemented; checkboxes are not updated. + **Goal:** Store the Criteo ID from Criteo's first-party cookie, as an `[[ec.identity_providers]]` entry with `kind = "criteo"`. No server call. **Architecture:** `crates/trusted-server-identity-providers/src/criteo.rs` holds `Criteo` (a `ProviderKind`), `CriteoSettings` and `CriteoProvider`. The provider implements only `IdentityProvider::read_request_cookies`; `build_request` always fails. @@ -19,11 +21,11 @@ No Lockr capability. ## Stored ID - One EID for the single configured source, one UID with `atype = 1`. -- Default bounded expiry; each request carrying the cookie refreshes it. +- Default bounded expiry; a cookie repeating the held ID is not rewritten, so the expiry renews only when the source is due. ## Not ported -- Criteo's `gum.criteo.com` sync call needs Criteo's third-party cookie, so it is not made; the first-party cookie is enough. +- Criteo has no public publisher S2S ID API; Criteo's `gum.criteo.com` sync call needs Criteo's third-party cookie, so it is not made; the first-party cookie is enough. ## Tasks diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-id5.md b/docs/superpowers/plans/2026-11-02-identity-provider-id5.md index a9a1b5305..3f0429360 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-id5.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-id5.md @@ -2,6 +2,8 @@ > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. +**Status:** Implemented; checkboxes are not updated. + **Goal:** Resolve the ID5 ID at the edge as an anonymous `[[ec.identity_providers]]` entry with `kind = "id5"`, storing the complete EIDs ID5 returns. **Architecture:** `crates/trusted-server-identity-providers/src/id5.rs` holds `Id5` (a `ProviderKind`), `Id5Settings` and `Id5Provider` (an `IdentityProvider`), plus one `build` arm in `lib.rs`. The provider is pure: `build_request` builds one JSON `POST`, and `parse_response` borrows EIDs from the body. The coordinator from PR A (`ec::coordinator`) applies consent, the source gate, validation, expiry and persistence. diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md b/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md index 378554e7c..4af83fefb 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md @@ -2,6 +2,8 @@ > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. +**Status:** Implemented; checkboxes are not updated. + **Goal:** Resolve `LiveIntent`'s IDs at the edge as a no-login `[[ec.identity_providers]]` entry with `kind = "liveintent"`, storing one EID per resolved attribute. **Architecture:** `crates/trusted-server-identity-providers/src/liveintent.rs` holds `LiveIntent` (a `ProviderKind`), `LiveIntentSettings` and `LiveIntentProvider` (an `IdentityProvider`), plus one `build` arm in `lib.rs`. The provider is pure: `build_request` builds one `GET` to the identity exchange, and `parse_response` maps returned attributes to sources. The coordinator from PR A applies consent, the source gate, validation, expiry and persistence. diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md b/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md index b17f97a65..22e9c07a3 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md @@ -2,9 +2,11 @@ > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. +**Status:** Implemented; checkboxes are not updated. + **Goal:** Store the LiveRamp RampID envelope the browser already sends in LiveRamp's own first-party cookie, as an `[[ec.identity_providers]]` entry with `kind = "liveramp"`. No login and no server call. -**Architecture:** `crates/trusted-server-identity-providers/src/liveramp.rs` holds `LiveRamp` (a `ProviderKind`), `LiveRampSettings` and `LiveRampProvider`. The provider implements only `IdentityProvider::read_request_cookies`; `build_request` always fails, so the coordinator never sends a lookup. The coordinator reads cookies only for due sources and applies consent, validation, expiry and persistence exactly as for lookup results. +**Architecture:** `crates/trusted-server-identity-providers/src/liveramp.rs` holds `LiveRamp` (a `ProviderKind`), `LiveRampSettings` and `LiveRampProvider`. The provider implements only `IdentityProvider::read_request_cookies`; `build_request` always fails, so the coordinator never sends a lookup. The coordinator reads cookies on every navigation (a cookie repeating the held UID is not rewritten) and applies consent, validation, expiry and persistence exactly as for lookup results. **Spec:** `docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md`, section 7. @@ -19,7 +21,7 @@ - One EID for the single configured source, one UID with `atype = 3`. - Expiry `ProviderExpiry::After(15 days)` through `bounded_expiry`. -- No refresh call: LiveRamp's server-side refresh contract is not confirmed, so the browser keeps refreshing the cookie. Revisit when it is. +- Optional server lookup (with `endpoint`): `GET /api/identity/v2/envelope` with `pid` (`placement_id`), `it=15`, `iv=:`, `atype=2`, consent as `ct`/`cv` (TCF `4`, else US Privacy `3`), `gpp` and `gpp_sid`; headers `Origin` (publisher) and `X-Forwarded-For` (routable visitor IP, else no lookup). `200 {"envelope"}` is stored like the cookie; `204` is an opt-out; a blank envelope is no match; other statuses fail. Without `endpoint`, no server call. LiveRamp must enable Custom ID for the placement and approve the origin. ## Lockr @@ -27,10 +29,12 @@ ## Not ported from the reference provider -- The reference refreshes a stored envelope server side (placement ID, approved countries, 15 or 30 days by region). It needs the visitor's region, which `LookupInput` does not carry, and its request and response shape are unconfirmed with LiveRamp, so this provider keeps reading the `_lr_env` cookie only. +- Hashed-email and hashed-phone lookups (`it=4`, `it=11`): out of scope (no logged-in experience). ## Tasks +- [x] Custom ID envelope lookup with `endpoint`, `placement_id` and `custom_id_namespace`, with tests. + - [ ] Add `cookie_capabilities`, `cookie_eid`, `is_usable_id` and `percent_decoded` to `common.rs`. - [ ] Add `liveramp.rs` and register the kind in `is_registered_kind` and `build`. - [ ] Unit tests: base64 and base64url envelopes, configured cookie name, missing, malformed and opt-out cookies ignored, one-source and cookie-name config checks, no lookup built. diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-lockr-capability.md b/docs/superpowers/plans/2026-11-02-identity-provider-lockr-capability.md index 3fc31556c..9254a2c2b 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-lockr-capability.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-lockr-capability.md @@ -2,6 +2,8 @@ > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. +**Status:** Implemented; checkboxes are not updated. + **Goal:** Read vendor IDs that Lockr AIM writes into first-party cookies on the publisher domain, as an explicit, reusable capability of the provider kinds Lockr delivers (ID5, LiveRamp, UID2), instead of ad-hoc cookie names per provider. **Architecture:** Everything lives in `crates/trusted-server-identity-providers/src/common.rs`; core only passes the configured provider ID to builds (`ProviderCommon::id`) for logging. diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-sharedid.md b/docs/superpowers/plans/2026-11-02-identity-provider-sharedid.md index 8a4e3e2c8..afd79ecb7 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-sharedid.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-sharedid.md @@ -2,9 +2,11 @@ > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. -**Goal:** Store the Prebid SharedID (publisher common ID) from its first-party cookie, as an `[[ec.identity_providers]]` entry with `kind = "sharedid"`. No server call. +**Status:** Implemented; checkboxes are not updated. -**Architecture:** `crates/trusted-server-identity-providers/src/sharedid.rs` holds `SharedId` (a `ProviderKind`), `SharedIdSettings` and `SharedIdProvider`. The provider implements only `IdentityProvider::read_request_cookies`; `build_request` always fails, so no lookup is sent. `common::device_eid` builds the agent type 1 EID. +**Goal:** Store the Prebid SharedID (publisher common ID) from its first-party cookie, as an `[[ec.identity_providers]]` entry with `kind = "sharedid"`. No server call. When no cookie ID and no live stored ID exist, TS generates a random UUID v4 into KV only (no cookie set); a later Prebid cookie replaces it. + +**Architecture:** `crates/trusted-server-identity-providers/src/sharedid.rs` holds `SharedId` (a `ProviderKind`), `SharedIdSettings` and `SharedIdProvider`. The provider implements `IdentityProvider::read_request_cookies` and `generate`; `build_request` always fails, so no lookup is sent. `common::device_eid` builds the agent type 1 EID. **Spec:** `docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md`, section 7. @@ -20,11 +22,13 @@ No Lockr capability. ## Stored ID - One EID for the single configured source, one UID with `atype = 1`, as Prebid sends `pubcid.org`. -- Default bounded expiry; each request carrying the cookie refreshes it. +- Cookie IDs: default bounded expiry; a cookie repeating the held ID is not rewritten, so the expiry renews only when the source is due. +- Generated IDs expire after 30 days (`MAX_TTL`), not the default 24 hours. ## Tasks - [ ] Add `sharedid.rs` and `common::device_eid`; register the kind in `is_registered_kind` and `build`. - [ ] Unit tests: `_sharedid` before `_pubcid`, opt-out cookie honoured, blank and opt-out IDs ignored. +- [ ] `generate` (UUID v4, 30-day expiry) + test. - [ ] Add a commented `kind = "sharedid"` example to `trusted-server.example.toml`. - [ ] Run `cargo test-fastly` and `cargo clippy-fastly`. diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md b/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md index ae929d867..4461c9dff 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md @@ -2,15 +2,17 @@ > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. -**Goal:** Store the UID2 advertising token the browser already sends in first-party cookies, as an `[[ec.identity_providers]]` entry with `kind = "uid2"`. No login, no server call and no server-side refresh. +**Status:** Implemented; checkboxes are not updated. -**Architecture:** `crates/trusted-server-identity-providers/src/uid2.rs` holds `Uid2` (a `ProviderKind`), `Uid2Settings` and `Uid2Provider`. The provider implements only `IdentityProvider::read_request_cookies`, using the cookie helpers in `common.rs`; `build_request` always fails, so no lookup is sent. +**Goal:** Store the UID2 advertising token from first-party cookies, as an `[[ec.identity_providers]]` entry with `kind = "uid2"`, and, when `endpoint` is set, refresh it server side. No login; tokens are never generated. + +**Architecture:** `crates/trusted-server-identity-providers/src/uid2.rs` holds `Uid2` (a `ProviderKind`), `Uid2Settings` and `Uid2Provider`. `read_request_cookies` reads the token, using the cookie helpers in `common.rs`; with an `endpoint`, `build_request` posts the stored refresh token to `POST /v2/token/refresh` and `parse_response` decrypts the AES-256-GCM reply (12-byte IV first) using the response key passed in `ProviderRequest::context`. Without an `endpoint`, `build_request` fails. **Spec:** `docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md`, section 7. ## Cookies -- `identity_cookie_name` (default `__uid_2`): percent-encoded JSON from the UID2 SDK. Only `advertising_token` and `identity_expires` (milliseconds) are deserialized. `refresh_token`, `refresh_from`, `refresh_expires` and `refresh_response_key` are never read, so they are never stored. +- `identity_cookie_name` (default `__uid_2`): percent-encoded JSON from the UID2 SDK. `advertising_token` and `identity_expires` (milliseconds) are read. With an `endpoint`, `refresh_token`, `refresh_response_key` and `refresh_from` are kept as private refresh state (never served, mirrored or forwarded; the admin view shows it as `[redacted]`); without it they are not stored. `refresh_expires` is not read. - `cookie_names` (default `["__uid2_advertising_token"]`): cookies holding a raw token, such as Prebid's, read in order only when the identity cookie yields nothing. - A missing token or expiry, a blank or opt-out token, or malformed JSON reads as no ID. @@ -19,7 +21,7 @@ - One EID for the single configured source, one UID with `atype = 3`. - From the identity cookie: `ProviderExpiry::At(identity_expires)`; the coordinator's validation drops an expired token. - From the raw token cookie: `bounded_expiry(None)`, the shared default TTL. -- No refresh: the browser SDK refreshes the identity and the next navigation with a due source picks up the new token. +- Refresh state `from = refresh_from`; the source becomes due then, and is refreshed only when the cookies no longer yield a usable token. A refresh is due only while the token is still live, so a failing refresh stops at its expiry; `rate_limit_per_hour` also bounds it. An `optout` reply removes the stored token. ## Lockr @@ -27,11 +29,14 @@ ## Not ported from the reference provider -- The reference generates a first token from a hashed email and refreshes it with the refresh token it kept from that call. Generation needs login, and refreshing with the SDK cookie's refresh token would spend the browser's single-use refresh token without a way to write the new `__uid_2` cookie back, so neither is ported. +- Token generation (needs a hashed email; out of scope). + +Risk: a server refresh spends the SDK cookie's refresh token without rewriting `__uid_2`, so the browser SDK's own refresh of that identity may then fail. That is acceptable because server refresh runs only when the browser sends no usable token. ## Tasks - [ ] Add `uid2.rs` and register the kind in `is_registered_kind` and `build`. - [ ] Unit tests: token and expiry from `__uid_2` with refresh fields not stored, raw token fallback, configured cookie names, missing, malformed and opt-out cookies ignored. +- [ ] Refresh: `endpoint`, AES-256-GCM decrypt with the response key from `ProviderRequest::context`, `optout` reply as `LookupOutcome::OptedOut`, and `refresh_state` tests. - [ ] Add a commented `kind = "uid2"` example to `trusted-server.example.toml`. - [ ] Run `cargo test-fastly` and `cargo clippy-fastly`. diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-unifiedid.md b/docs/superpowers/plans/2026-11-02-identity-provider-unifiedid.md index ce53d5fb5..927ee3867 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-unifiedid.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-unifiedid.md @@ -2,6 +2,8 @@ > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. +**Status:** Implemented; checkboxes are not updated. + **Goal:** Store The Trade Desk ID (TDID) from Prebid's Unified ID first-party cookie, as an `[[ec.identity_providers]]` entry with `kind = "unifiedid"`. No server call. **Architecture:** `crates/trusted-server-identity-providers/src/unifiedid.rs` holds `UnifiedId` (a `ProviderKind`), `UnifiedIdSettings` and `UnifiedIdProvider`. The provider implements only `IdentityProvider::read_request_cookies` and keeps EID-level fields (`EidLevelFields::Preserved`). @@ -19,9 +21,13 @@ No Lockr capability. ## Stored ID - One EID shaped as Prebid sends it: `atype = 1`, UID `ext` `{"rtiPartner":"TDID"}`, `inserter` and `matcher` `adserver.org`, `mm = 4`. -- Default bounded expiry; each request carrying the cookie refreshes it. +- Default bounded expiry; a cookie repeating the held ID is not rewritten, so the expiry renews only when the source is due. - The source must not also be claimed by a liveintent `thetradedesk` attribute; pick one owner. +## Not ported + +- The Trade Desk has no public publisher S2S ID API; only the Prebid cookie is read. + ## Tasks - [ ] Add `unifiedid.rs`; register the kind in `is_registered_kind` and `build`. diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 565704378..d909a1d85 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -122,9 +122,7 @@ pull_sync_concurrency = 3 # pull_sync_enabled = false # default false # Optional server-side identity providers (Fastly only), looked up on publisher -# navigations before the origin fetch. Their budgets are set in [ec] above. This -# release registers no provider kind, so `ts config push` rejects an enabled -# provider and every runtime refuses to start with one. +# navigations before the origin fetch. Their budgets are set in [ec] above. # # [[ec.identity_providers]] # id = "example-provider" @@ -140,10 +138,13 @@ pull_sync_concurrency = 3 # these sources to bidders, including browser IDs, even while disabled. # tcf_vendor_id = 9999 # "page" (default) collects before the response; "after_response" collects -# after it is sent, only for an EC issued on an earlier request. +# after it is sent, only for an EC issued on an earlier request. Keep +# cookie-only kinds on "page". # collect = "page" # Optional: most lookups per EC hash (client IP) per hour. On Fastly this is # enforced per minute with a minimum of one, so values below 60 behave like 60. +# Useful for kinds whose unresolved sources stay due, such as LiveIntent +# partner IDs. # rate_limit_per_hour = 60 # Secret Store key name, resolved like other secret references. # [ec.identity_providers.secrets] @@ -209,19 +210,28 @@ pull_sync_concurrency = 3 # resolve = ["nonId", "magnite"] # cookie_names = ["_lc2_fpi"] # -# LiveRamp (kind "liveramp"): no lookup and no endpoint. Reads the RampID -# envelope from LiveRamp's own first-party `_lr_env` cookie (ATS) or the bare -# envelope Prebid's identityLink module stores in `idl_env`, and keeps it for -# 15 days. Takes exactly one source; use the one LiveRamp gives you. Supports -# the Lockr capability. +# LiveRamp (kind "liveramp"): reads the RampID envelope from LiveRamp's own +# first-party `_lr_env` cookie (ATS) or the bare envelope Prebid's +# identityLink module stores in `idl_env`, and keeps it for 15 days. Takes +# exactly one source; use the one LiveRamp gives you. Supports the Lockr +# capability. Optional server lookup: with an `endpoint` (LiveRamp's ATS +# envelope API, `/api/identity/v2/envelope`), a visitor without an envelope +# cookie is looked up by Custom ID: the stable part of the EC ID, sent as +# `:` with the visitor IP and consent. LiveRamp must +# enable Custom ID for `placement_id` and approve the publisher origin; both +# settings are required with an endpoint. No login or hashed email is used. # [[ec.identity_providers]] # id = "liveramp" # kind = "liveramp" # enabled = false +# endpoint = "https://liveramp.example.com/api/identity/v2/envelope" +# allowed_domains = ["liveramp.example.com"] # sources = ["liveramp.example.com"] # tcf_vendor_id = 9999 # [ec.identity_providers.settings] # cookie_names = ["_lr_env", "idl_env"] +# placement_id = 14 +# custom_id_namespace = "ana-example" # [ec.identity_providers.settings.lockr] # enabled = true # cookie_names = ["_lr_env"] @@ -238,16 +248,24 @@ pull_sync_concurrency = 3 # [ec.identity_providers.settings] # cookie_names = ["connectId"] # -# UID2 (kind "uid2"): no lookup and no endpoint. Reads the advertising token -# from the UID2 SDK's `__uid_2` cookie until its `identity_expires`, falling -# back to Prebid's raw `__uid2_advertising_token` cookie. Never stores or uses -# the refresh token. Takes exactly one source. `cookie_names` lists the raw -# token cookies, read in order after `identity_cookie_name`. Supports the -# Lockr capability; Lockr cookies are read last, as raw tokens. +# UID2 (kind "uid2"): reads the advertising token from the UID2 SDK's +# `__uid_2` cookie until its `identity_expires`, falling back to Prebid's raw +# `__uid2_advertising_token` cookie. Takes exactly one source. `cookie_names` +# lists the raw token cookies, read in order after `identity_cookie_name`. +# Supports the Lockr capability; Lockr cookies are read last, as raw tokens. +# Optional server refresh: set `endpoint` to the UID2 operator's +# `/v2/token/refresh` URL. The `__uid_2` cookie's refresh token and response +# key are then kept privately in the row (never served), and the token is +# refreshed server side from `refresh_from`, with no API key and no email. +# Tokens are never generated: that needs an email or phone number. Without +# `endpoint`, the refresh token is never stored. The authenticated +# `/_ts/admin/ec/{id}` view shows refresh state as "[redacted]". # [[ec.identity_providers]] # id = "uid2" # kind = "uid2" # enabled = false +# endpoint = "https://uid2.example.com/v2/token/refresh" +# allowed_domains = ["uid2.example.com"] # sources = ["uid2.example.com"] # [ec.identity_providers.settings] # identity_cookie_name = "__uid_2" @@ -259,7 +277,10 @@ pull_sync_concurrency = 3 # Prebid SharedID (kind "sharedid"): no lookup and no endpoint. Reads the # publisher common ID from Prebid's first-party `_sharedid` cookie, falling # back to the legacy `_pubcid`, as a device-based (atype 1) ID. The -# `_pubcid_optout` cookie suppresses it. Takes exactly one source. +# `_pubcid_optout` cookie suppresses it. Takes exactly one source. When no +# cookie holds an ID and none is stored, Trusted Server generates a random +# UUID and stores it in the row only (no cookie is set); a later Prebid cookie +# replaces it. # [[ec.identity_providers]] # id = "sharedid" # kind = "sharedid" From 0dc72b3f203e8193adee02ced0f4790679d50f17 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 03:07:00 +0530 Subject: [PATCH 24/72] Build identity providers when validating the app config ts config validate, push and diff now build every enabled identity provider of a registered kind in both collection phases, so a provider-specific setting error (such as a liveramp endpoint without custom_id_namespace) fails the command before deploy. At runtime the same error disables every provider for each request. Partner tokens are replaced with placeholders for the check, as core deploy validation already does. Signed-off-by: dhruv8sh --- Cargo.lock | 2 + crates/trusted-server-cli/Cargo.toml | 2 + .../src/checked_app_config.rs | 170 ++++++++++++++++++ crates/trusted-server-cli/src/lib.rs | 2 + crates/trusted-server-cli/src/run.rs | 8 +- crates/trusted-server-core/src/config.rs | 3 +- .../src/lib.rs | 98 ++++++++++ docs/guide/configuration.md | 7 + 8 files changed, 287 insertions(+), 5 deletions(-) create mode 100644 crates/trusted-server-cli/src/checked_app_config.rs diff --git a/Cargo.lock b/Cargo.lock index a62c86afe..7c73c0256 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6739,8 +6739,10 @@ dependencies = [ "tracing", "trusted-server-adapter-axum", "trusted-server-core", + "trusted-server-identity-providers", "url", "uuid", + "validator", "webpki-roots", "which", "x509-parser", diff --git a/crates/trusted-server-cli/Cargo.toml b/crates/trusted-server-cli/Cargo.toml index 4d6af83fd..70b85f497 100644 --- a/crates/trusted-server-cli/Cargo.toml +++ b/crates/trusted-server-cli/Cargo.toml @@ -57,8 +57,10 @@ toml = { workspace = true } toml_edit = { workspace = true } tracing = { workspace = true } trusted-server-core = { workspace = true } +trusted-server-identity-providers = { workspace = true } url = { workspace = true } uuid = { workspace = true } +validator = { workspace = true } which = { workspace = true } # `ts dev lint domains` and `ts dev install-hooks` drive all git access through diff --git a/crates/trusted-server-cli/src/checked_app_config.rs b/crates/trusted-server-cli/src/checked_app_config.rs new file mode 100644 index 000000000..bc51ecbf0 --- /dev/null +++ b/crates/trusted-server-cli/src/checked_app_config.rs @@ -0,0 +1,170 @@ +//! The app-config type that `ts config push`, `validate` and `diff` check. +//! +//! [`TrustedServerAppConfig`] validation lives in core, which cannot build +//! identity providers because the provider crate depends on core. At runtime +//! a provider that fails to build disables every provider for the request, so +//! this wrapper adds [`validate_providers`] to the config checks: a +//! provider-specific setting error then fails the command before deploy. + +use std::borrow::Cow; + +use edgezero_core::app_config::{AppConfigMeta, SecretField}; +use error_stack::{AttachmentKind, FrameKind, Report}; +use serde::{Deserialize, Serialize}; +use trusted_server_core::config::TrustedServerAppConfig; +use trusted_server_core::ec::provider::ProviderConfigError; +use trusted_server_identity_providers::validate_providers; +use validator::{Validate, ValidationError, ValidationErrors}; + +/// Field that identity-provider build errors are reported under. +const PROVIDER_VALIDATION_FIELD: &str = "identity_providers"; + +/// [`TrustedServerAppConfig`] whose validation also builds every enabled +/// identity provider. +#[derive(Serialize, Deserialize)] +#[serde(transparent)] +pub(crate) struct CheckedAppConfig(TrustedServerAppConfig); + +impl Validate for CheckedAppConfig { + fn validate(&self) -> Result<(), ValidationErrors> { + let mut errors = self.0.validate().err().unwrap_or_default(); + if let Err(report) = validate_providers(self.0.settings()) { + let mut error = ValidationError::new("trusted_server_identity_providers"); + error.message = Some(Cow::Owned(provider_error_message(&report))); + errors.add(PROVIDER_VALIDATION_FIELD, error); + } + if errors.errors().is_empty() { + Ok(()) + } else { + Err(errors) + } + } +} + +impl AppConfigMeta for CheckedAppConfig { + fn secret_fields() -> Vec { + TrustedServerAppConfig::secret_fields() + } +} + +/// Joins the report's context with every printable attachment and underlying +/// error, which name the failing provider and setting (for example the +/// settings parser's `unknown field` error). +fn provider_error_message(report: &Report) -> String { + let mut details: Vec = Vec::new(); + for frame in report.frames() { + let detail = match frame.kind() { + FrameKind::Attachment(AttachmentKind::Printable(printable)) => printable.to_string(), + FrameKind::Context(context) if !frame.is::() => { + context.to_string() + } + _ => continue, + }; + if !details.contains(&detail) { + details.push(detail); + } + } + if details.is_empty() { + report.current_context().to_string() + } else { + format!("{}: {}", report.current_context(), details.join("; ")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const EXAMPLE_CONFIG: &str = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../trusted-server.example.toml" + )); + + fn with_liveramp(settings: &str) -> CheckedAppConfig { + let config = format!( + "{EXAMPLE_CONFIG}\n\ + [[ec.partners]]\n\ + name = \"liveramp\"\n\ + source_domain = \"liveramp.example.com\"\n\ + bidstream_enabled = true\n\ + \n\ + [[ec.identity_providers]]\n\ + id = \"liveramp\"\n\ + kind = \"liveramp\"\n\ + endpoint = \"https://envelope.example.com/api/identity/v2/envelope\"\n\ + allowed_domains = [\"envelope.example.com\"]\n\ + sources = [\"liveramp.example.com\"]\n\ + [ec.identity_providers.settings]\n\ + {settings}\n" + ); + toml::from_str(&config).expect("should parse app config") + } + + /// The identity-provider validation message, when there is one. + /// + /// The example config fails unrelated core checks (placeholder publisher + /// values, unresolved secrets), so tests look only at this field. + fn provider_error(config: &CheckedAppConfig) -> Option { + let errors = config.validate().err()?; + errors + .field_errors() + .get(PROVIDER_VALIDATION_FIELD) + .and_then(|errors| errors.first()) + .and_then(|error| error.message.as_deref()) + .map(str::to_owned) + } + + #[test] + fn reports_no_provider_error_for_the_example_config() { + let config: CheckedAppConfig = + toml::from_str(EXAMPLE_CONFIG).expect("should parse the example config"); + + assert_eq!( + provider_error(&config), + None, + "should build the example config's providers" + ); + } + + #[test] + fn reports_no_provider_error_for_a_provider_that_builds() { + let config = with_liveramp("placement_id = 14\ncustom_id_namespace = \"ana-example\""); + + assert_eq!( + provider_error(&config), + None, + "should accept a complete liveramp lookup" + ); + } + + #[test] + fn rejects_a_provider_that_cannot_be_built_and_keeps_core_errors() { + let config = with_liveramp("placement_id = 14"); + + let message = provider_error(&config).expect("should report the provider error"); + let errors = config.validate().expect_err("should fail validation"); + + assert!( + message.contains("custom_id_namespace"), + "should name the missing setting, got: {message}" + ); + assert!( + errors.errors().contains_key("trusted_server"), + "should keep the core validation errors" + ); + } + + #[test] + fn names_the_unknown_setting_in_a_provider_settings_typo() { + let config = with_liveramp( + "placement_id = 14\ncustom_id_namespace = \"ana-example\"\ncookie_nmes = [\"_lr_env\"]", + ); + + let message = provider_error(&config).expect("should report the provider error"); + + assert!( + message.contains("cookie_nmes"), + "should name the unknown setting, got: {message}" + ); + } +} diff --git a/crates/trusted-server-cli/src/lib.rs b/crates/trusted-server-cli/src/lib.rs index 7f6b9e6e7..60dd54ce8 100644 --- a/crates/trusted-server-cli/src/lib.rs +++ b/crates/trusted-server-cli/src/lib.rs @@ -3,6 +3,8 @@ mod ad_templates; #[cfg(not(target_arch = "wasm32"))] mod app_config; #[cfg(not(target_arch = "wasm32"))] +mod checked_app_config; +#[cfg(not(target_arch = "wasm32"))] mod error; #[cfg(not(target_arch = "wasm32"))] mod fs_util; diff --git a/crates/trusted-server-cli/src/run.rs b/crates/trusted-server-cli/src/run.rs index 992452da3..28a466673 100644 --- a/crates/trusted-server-cli/src/run.rs +++ b/crates/trusted-server-cli/src/run.rs @@ -6,9 +6,9 @@ use edgezero_cli::args::{ ActiveVersionArgs, AuthArgs, BuildArgs, ConfigDiffArgs, ConfigGcArgs, ConfigPushArgs, ConfigValidateArgs, DeployArgs, HealthcheckArgs, ProvisionArgs, RollbackArgs, ServeArgs, }; -use trusted_server_core::config::TrustedServerAppConfig; use trusted_server_core::constants::TS_GIT_VERSION; +use crate::checked_app_config::CheckedAppConfig; use crate::commands::audit::{AuditArgs, run_audit}; use crate::commands::config::ad_templates::{AdTemplatesCommand, run_ad_templates}; use crate::commands::config::init::{ConfigInitArgs, run_config_init}; @@ -144,7 +144,7 @@ fn dispatch(args: Args) -> Result { run_config_init(&args).map(|()| RunOutcome::Success) } Command::Config(ConfigCommand::Diff(args)) => { - match edgezero_cli::run_config_diff_typed::(&args) { + match edgezero_cli::run_config_diff_typed::(&args) { Ok(edgezero_cli::DiffExit { code: 0 }) => Ok(RunOutcome::Success), Ok(edgezero_cli::DiffExit { code: 1 }) => Ok(RunOutcome::AssertionFailed), Ok(edgezero_cli::DiffExit { code }) => process::exit(code), @@ -155,11 +155,11 @@ fn dispatch(args: Args) -> Result { edgezero_cli::run_config_gc(&args).map(|()| RunOutcome::Success) } Command::Config(ConfigCommand::Push(args)) => { - edgezero_cli::run_config_push_typed::(&args) + edgezero_cli::run_config_push_typed::(&args) .map(|()| RunOutcome::Success) } Command::Config(ConfigCommand::Validate(args)) => { - edgezero_cli::run_config_validate_typed::(&args) + edgezero_cli::run_config_validate_typed::(&args) .map(|()| RunOutcome::Success) } Command::Deploy(args) => edgezero_cli::run_deploy(&args).map(|()| RunOutcome::Success), diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 057ebe9e2..f729a22f2 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -426,7 +426,8 @@ fn validate_gpt_diagnostics_config(settings: &Settings) -> Result<(), Report Ec { +#[must_use] +pub fn with_placeholder_partner_tokens(ec: &Ec) -> Ec { let mut ec = ec.clone(); for (index, partner) in ec.partners.iter_mut().enumerate() { if partner.api_token.is_some() { diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index e6cb80a11..876ae0132 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -25,6 +25,7 @@ use std::sync::Arc; use std::time::Duration; use error_stack::Report; +use trusted_server_core::config::with_placeholder_partner_tokens; use trusted_server_core::ec::coordinator::{ProviderCoordinator, RegisteredProvider}; use trusted_server_core::ec::provider::{ IdentityProvider, ProviderConfigError, ProviderKind as _, build_kind, @@ -158,6 +159,40 @@ pub fn build_after_response_coordinator( ) } +/// Builds every enabled provider of both collection phases, as the adapters +/// do on each navigation, so a provider-specific setting error is reported +/// before deploy. +/// +/// At runtime such an error disables every provider for the request, so +/// deploy-time config validation (`ts config push`, `validate` and `diff`) +/// calls this. Partner tokens may still be secret key names at that point; +/// they are replaced with placeholders for the check. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError`] for the first provider that cannot be +/// built or fails the coordinator's configuration checks. +pub fn validate_providers(settings: &Settings) -> Result<(), Report> { + let ec = with_placeholder_partner_tokens(&settings.ec); + let partners = Arc::new( + partner_registry(&ec) + .map_err(|report| report.change_context(ProviderConfigError::Invalid))?, + ); + build_phase_coordinator( + settings, + Arc::clone(&partners), + CollectTiming::Page, + Duration::from_millis(settings.ec.identity_budget_ms), + )?; + build_phase_coordinator( + settings, + partners, + CollectTiming::AfterResponse, + Duration::from_millis(settings.ec.identity_after_response_budget_ms), + )?; + Ok(()) +} + fn build_phase_coordinator( settings: &Settings, partners: Arc, @@ -365,6 +400,69 @@ mod tests { settings } + fn liveramp_provider(collect: &str, settings: &str) -> IdentityProviderConfig { + toml::from_str(&format!( + "id = \"liveramp\"\nkind = \"liveramp\"\nsources = [\"liveramp.example.com\"]\n\ + endpoint = \"https://envelope.example.com/api/identity/v2/envelope\"\n\ + allowed_domains = [\"envelope.example.com\"]\ncollect = \"{collect}\"\n\ + [settings]\n{settings}" + )) + .expect("should parse liveramp") + } + + fn settings_with_liveramp(collect: &str, settings: &str) -> Settings { + let mut configured = settings_with_cookie_providers("page", "page"); + configured.ec.partners.push( + toml::from_str( + "name = \"liveramp\"\nsource_domain = \"liveramp.example.com\"\nbidstream_enabled = true", + ) + .expect("should parse partner"), + ); + configured + .ec + .identity_providers + .push(liveramp_provider(collect, settings)); + configured + } + + #[test] + fn validate_providers_accepts_buildable_providers_in_both_phases() { + let page = settings_with_liveramp( + "page", + "placement_id = 14\ncustom_id_namespace = \"ana-example\"", + ); + let after = settings_with_liveramp( + "after_response", + "placement_id = 14\ncustom_id_namespace = \"ana-example\"", + ); + + validate_providers(&page).expect("should accept a buildable page provider"); + validate_providers(&after).expect("should accept a buildable after-response provider"); + } + + #[test] + fn validate_providers_rejects_a_provider_that_cannot_be_built() { + for collect in ["page", "after_response"] { + let settings = settings_with_liveramp(collect, "placement_id = 14"); + + assert!( + validate_providers(&settings).is_err(), + "should reject a {collect} liveramp endpoint without a namespace" + ); + } + } + + #[test] + fn validate_providers_accepts_partner_tokens_that_are_still_key_names() { + let mut settings = settings_with_cookie_providers("page", "page"); + settings.ec.partners[0] = toml::from_str( + "name = \"pubcid.org\"\nsource_domain = \"pubcid.org\"\nbidstream_enabled = true\napi_token = \"partner_token_key\"", + ) + .expect("should parse partner with a token key name"); + + validate_providers(&settings).expect("should not resolve partner token key names"); + } + #[test] fn parses_an_optional_hourly_rate_limit() { let limited: IdentityProviderConfig = toml::from_str( diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index c9235fe47..092626f41 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1021,6 +1021,13 @@ build, and an older writer drops a row's `eids` and provider opt-outs when it rewrites the row. ::: +`ts config validate`, `ts config push` and `ts config diff` also build every +enabled provider of a registered kind, so a provider-specific setting error +(for example a `liveramp` `endpoint` without `placement_id` and +`custom_id_namespace`) fails the command before deploy. At runtime the same +error would disable every identity provider for each request, with an error +log. + ### Field Details #### `passphrase` From d12c9a51f37dd20df6a48c3743fd94497f3e8b61 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 16:26:02 +0530 Subject: [PATCH 25/72] Send ID5 GPP section IDs as an array and use the gs/v2 fetch path Signed-off-by: dhruv8sh --- .../src/id5.rs | 20 +++++++++++-------- .../src/lib.rs | 2 +- trusted-server.example.toml | 2 +- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/crates/trusted-server-identity-providers/src/id5.rs b/crates/trusted-server-identity-providers/src/id5.rs index f63e37509..25ce505aa 100644 --- a/crates/trusted-server-identity-providers/src/id5.rs +++ b/crates/trusted-server-identity-providers/src/id5.rs @@ -53,7 +53,7 @@ use serde_json::value::RawValue; use crate::common::{ CookieReader, LockrSettings, ReadsLockr, VendorEid, bounded_expiry, is_routable_ip, - is_usable_id, join_section_ids, percent_decoded, + is_usable_id, percent_decoded, }; /// The UID ID5 returns for a visitor who opted out. @@ -169,7 +169,7 @@ struct FetchBody<'a> { #[serde(skip_serializing_if = "Option::is_none")] gpp_string: Option<&'a str>, #[serde(skip_serializing_if = "Option::is_none")] - gpp_sid: Option, + gpp_sid: Option<&'a [u16]>, tml: &'a str, #[serde(skip_serializing_if = "Option::is_none")] ua: Option<&'a str>, @@ -341,7 +341,7 @@ impl IdentityProvider for Id5Provider { gdpr_consent: consent.raw_tc_string.as_deref(), us_privacy: consent.raw_us_privacy.as_deref(), gpp_string: consent.raw_gpp_string.as_deref(), - gpp_sid: consent.gpp_section_ids.as_deref().map(join_section_ids), + gpp_sid: consent.gpp_section_ids.as_deref(), tml: &origin, ua: lookup.client.user_agent, ip: lookup @@ -460,7 +460,7 @@ mod tests { let config: IdentityProviderConfig = serde_json::from_value(json!({ "id": "id5", "kind": "id5", - "endpoint": "https://id5.example.com/g/v2", + "endpoint": "https://id5.example.com/gs/v2", "allowed_domains": ["id5.example.com"], "sources": ["id5.example.com", "id5-link.example.com"], "settings": { "partner_id": 1234 }, @@ -515,7 +515,7 @@ mod tests { assert_eq!(request.method, Method::POST, "should post"); assert_eq!( request.url.as_str(), - "https://id5.example.com/g/v2/1234.json", + "https://id5.example.com/gs/v2/1234.json", "should call the partner's fetch URL" ); assert_eq!( @@ -526,7 +526,11 @@ mod tests { assert_eq!(body["partner"], 1234, "should send the partner"); assert_eq!(body["gdpr"], 0, "should send that GDPR does not apply"); assert_eq!(body["us_privacy"], "1YNN", "should send US privacy"); - assert_eq!(body["gpp_sid"], "7,8", "should join GPP section IDs"); + assert_eq!( + body["gpp_sid"], + json!([7, 8]), + "should send GPP section IDs as an array" + ); assert_eq!(body["ip"], "11.22.33.44", "should send the client IP"); assert_eq!( body["ua"], "ExampleBrowser/1.0", @@ -777,7 +781,7 @@ mod tests { let config: IdentityProviderConfig = serde_json::from_value(json!({ "id": "id5", "kind": "id5", - "endpoint": "https://id5.example.com/g/v2", + "endpoint": "https://id5.example.com/gs/v2", "allowed_domains": ["id5.example.com"], "sources": ["id5.example.com"], "settings": { "partner_id": 1234, "cookie_names": ["example_lockr_id5", "id5id"] }, @@ -807,7 +811,7 @@ mod tests { let config: IdentityProviderConfig = serde_json::from_value(json!({ "id": "id5", "kind": "id5", - "endpoint": "https://id5.example.com/g/v2", + "endpoint": "https://id5.example.com/gs/v2", "allowed_domains": ["id5.example.com"], "sources": ["id5.example.com"], "settings": { diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index 876ae0132..21114d104 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -254,7 +254,7 @@ mod tests { for (kind, endpoint, sources, settings) in [ ( "id5", - Some("https://id5.example.com/g/v2"), + Some("https://id5.example.com/gs/v2"), json!(["id5.example.com"]), json!({ "partner_id": 1234 }), ), diff --git a/trusted-server.example.toml b/trusted-server.example.toml index d909a1d85..609f185d9 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -167,7 +167,7 @@ pull_sync_concurrency = 3 # id = "id5" # kind = "id5" # enabled = false -# endpoint = "https://id5.example.com/g/v2" +# endpoint = "https://id5.example.com/gs/v2" # allowed_domains = ["id5.example.com"] # sources = ["id5.example.com"] # tcf_vendor_id = 9999 From d45022c126c728d88341bb834c25bb704eaeb79e Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 18:36:36 +0530 Subject: [PATCH 26/72] Send the RFC 3339 timestamp the ID5 gs/v2 fetch requires Signed-off-by: dhruv8sh --- Cargo.lock | 1 + .../trusted-server-identity-providers/Cargo.toml | 1 + .../trusted-server-identity-providers/src/id5.rs | 14 ++++++++++++++ 3 files changed, 16 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 7c73c0256..66e402766 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6807,6 +6807,7 @@ dependencies = [ "aes-gcm", "base64", "bytes", + "chrono", "error-stack", "http", "httpdate", diff --git a/crates/trusted-server-identity-providers/Cargo.toml b/crates/trusted-server-identity-providers/Cargo.toml index da9e6a073..d267ebc66 100644 --- a/crates/trusted-server-identity-providers/Cargo.toml +++ b/crates/trusted-server-identity-providers/Cargo.toml @@ -16,6 +16,7 @@ workspace = true [dependencies] aes-gcm = { workspace = true } base64 = { workspace = true } +chrono = { workspace = true } bytes = { workspace = true } error-stack = { workspace = true } http = { workspace = true } diff --git a/crates/trusted-server-identity-providers/src/id5.rs b/crates/trusted-server-identity-providers/src/id5.rs index 25ce505aa..25f2d5af8 100644 --- a/crates/trusted-server-identity-providers/src/id5.rs +++ b/crates/trusted-server-identity-providers/src/id5.rs @@ -36,10 +36,12 @@ use std::num::NonZeroUsize; use std::time::Duration; use bytes::Bytes; +use chrono::{DateTime, SecondsFormat, Utc}; use error_stack::{Report, ResultExt as _}; use http::header::{CONTENT_TYPE, ORIGIN}; use http::{HeaderMap, HeaderValue, Method}; use serde::{Deserialize, Serialize}; +use trusted_server_core::ec::current_timestamp; use trusted_server_core::ec::eid_types::SourceDomain; use trusted_server_core::ec::provider::{ CookieRead, EidLevelFields, EidShape, IdentityProvider, LookupInput, LookupOutcome, @@ -56,6 +58,14 @@ use crate::common::{ is_usable_id, percent_decoded, }; +/// Formats the current time as the RFC 3339 `ts` the `/gs/v2` fetch requires. +fn request_timestamp() -> Result> { + let now = i64::try_from(current_timestamp()).change_context(ProviderError::Request)?; + DateTime::::from_timestamp(now, 0) + .map(|time| time.to_rfc3339_opts(SecondsFormat::Secs, true)) + .ok_or_else(|| Report::new(ProviderError::Request).attach("current time is out of range")) +} + /// The UID ID5 returns for a visitor who opted out. const OPTED_OUT_UID: &str = "0"; @@ -171,6 +181,7 @@ struct FetchBody<'a> { #[serde(skip_serializing_if = "Option::is_none")] gpp_sid: Option<&'a [u16]>, tml: &'a str, + ts: String, #[serde(skip_serializing_if = "Option::is_none")] ua: Option<&'a str>, #[serde(skip_serializing_if = "Option::is_none")] @@ -343,6 +354,7 @@ impl IdentityProvider for Id5Provider { gpp_string: consent.raw_gpp_string.as_deref(), gpp_sid: consent.gpp_section_ids.as_deref(), tml: &origin, + ts: request_timestamp()?, ua: lookup.client.user_agent, ip: lookup .client @@ -524,6 +536,8 @@ mod tests { "should send the publisher origin" ); assert_eq!(body["partner"], 1234, "should send the partner"); + let ts = body["ts"].as_str().expect("should send a ts string"); + DateTime::parse_from_rfc3339(ts).expect("should send ts as an RFC 3339 timestamp"); assert_eq!(body["gdpr"], 0, "should send that GDPR does not apply"); assert_eq!(body["us_privacy"], "1YNN", "should send US privacy"); assert_eq!( From 3aa6910a8879d87631f89616829ac9dff80b70b3 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 18:36:36 +0530 Subject: [PATCH 27/72] Log the HTTP status of a failed identity lookup Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/coordinator/mod.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 9a51a4fb3..892b400b9 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -708,6 +708,14 @@ fn log_outcome(lookup: &ProviderLookup) { LookupStatus::Found { kept, dropped } => (kept, dropped.total()), _ => (0, 0), }; + if let LookupStatus::HttpError(status) = lookup.status { + log::info!( + "Identity lookup: provider={} outcome={} status={status}", + lookup.provider, + lookup.status.metric_name() + ); + return; + } log::info!( "Identity lookup: provider={} outcome={} kept={kept} dropped={dropped}", lookup.provider, From 8520709cd2c556690c672fed4553c53cae5a7f2b Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 18:49:32 +0530 Subject: [PATCH 28/72] Treat an ID5 fetch reply with an opt-out UID as an opt-out Signed-off-by: dhruv8sh --- .../src/id5.rs | 42 ++++++++++++++----- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/crates/trusted-server-identity-providers/src/id5.rs b/crates/trusted-server-identity-providers/src/id5.rs index 25f2d5af8..00bfa6eca 100644 --- a/crates/trusted-server-identity-providers/src/id5.rs +++ b/crates/trusted-server-identity-providers/src/id5.rs @@ -24,7 +24,8 @@ //! usable UID is ignored and the lookup runs as usual. A cookie holding //! ID5's opt-out (a bare `0`, or a reply whose `universal_uid` is `0`) //! reports an opt-out: the lookup is skipped and stored ID5 UIDs are -//! removed. +//! removed. A fetch reply whose `universal_uid` is `0` reports the same +//! opt-out. //! //! The request field names for the client IP and user agent follow ID5's //! server-to-server integration and should be confirmed with ID5 for each @@ -386,6 +387,9 @@ impl IdentityProvider for Id5Provider { return Err(Report::new(ProviderError::InvalidResponse) .attach(format!("ID5 returned status {}", response.status))); } + if std::str::from_utf8(response.body).is_ok_and(is_opt_out_cookie) { + return Ok(LookupOutcome::OptedOut); + } let eids = self .parse_reply(response.body) .change_context(ProviderError::InvalidResponse)?; @@ -410,8 +414,8 @@ impl IdentityProvider for Id5Provider { /// The ID5 opt-out UID. const OPT_OUT_UID: &str = "0"; -/// Whether one ID5 cookie value records an opt-out: a bare `0`, or a reply -/// whose `universal_uid` is `0`. +/// Whether one ID5 cookie value or fetch reply records an opt-out: a bare +/// `0`, or a reply whose `universal_uid` is `0`. fn is_opt_out_cookie(value: &str) -> bool { let decoded = percent_decoded(value); let decoded = decoded.trim(); @@ -674,17 +678,33 @@ mod tests { } #[test] - fn reports_no_match_for_an_opted_out_or_empty_reply() { + fn reports_no_match_for_an_empty_reply() { let provider = provider(); let headers = HeaderMap::new(); + let body = serde_json::to_vec(&json!({})).expect("should encode body"); - for body in [json!({ "universal_uid": "0" }), json!({})] { - let body = serde_json::to_vec(&body).expect("should encode body"); - assert!( - matches!(respond(&*provider, &headers, &body), LookupOutcome::NoMatch), - "should report no match" - ); - } + assert!( + matches!(respond(&*provider, &headers, &body), LookupOutcome::NoMatch), + "should report no match" + ); + } + + #[test] + fn reports_an_opt_out_reply() { + let provider = provider(); + let headers = HeaderMap::new(); + let body = serde_json::to_vec(&json!({ + "universal_uid": "0", + "cache_control": { "max_age_sec": 7200 }, + })) + .expect("should encode body"); + + let outcome = respond(&*provider, &headers, &body); + + assert!( + matches!(outcome, LookupOutcome::OptedOut), + "should report an opt-out for a universal_uid of 0" + ); } #[test] From 7e9aa8e459e4471463030efcee8f31c011682a4d Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 18:49:59 +0530 Subject: [PATCH 29/72] Omit empty LiveIntent segments from the nonId EID Signed-off-by: dhruv8sh --- .../src/liveintent.rs | 45 ++++++++++++++++--- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/crates/trusted-server-identity-providers/src/liveintent.rs b/crates/trusted-server-identity-providers/src/liveintent.rs index 339a5def7..2579e9db0 100644 --- a/crates/trusted-server-identity-providers/src/liveintent.rs +++ b/crates/trusted-server-identity-providers/src/liveintent.rs @@ -10,8 +10,8 @@ //! As in Prebid's `LiveIntent` module, every attribute other than `nonId` is //! a partner ID and its UID carries `ext.provider` naming `LiveIntent`; the //! Trade Desk ID also carries `ext.rtiPartner` and its EID names -//! `LiveIntent` as `matcher`. `segments` in the reply become the `nonId` -//! EID's `ext`. +//! `LiveIntent` as `matcher`. A non-empty `segments` array in the reply +//! becomes the `nonId` EID's `ext`. //! //! This kind has no Lockr capability: Lockr AIM is not documented to //! deliver `LiveIntent` IDs, so a `lockr` settings table is rejected. @@ -325,9 +325,16 @@ impl IdentityProvider for LiveIntentProvider { let reply: HashMap = serde_json::from_slice(response.body).change_context(ProviderError::InvalidResponse)?; let expiry = expiry(response.headers); - let segments = reply.get("segments").and_then(|segments| { - RawValue::from_string(format!(r#"{{"segments":{}}}"#, segments.get())).ok() - }); + // Only a non-empty segment array is kept, as Prebid does. + let segments = reply + .get("segments") + .filter(|segments| { + serde_json::from_str::>(segments.get()) + .is_ok_and(|segments| !segments.is_empty()) + }) + .and_then(|segments| { + RawValue::from_string(format!(r#"{{"segments":{}}}"#, segments.get())).ok() + }); let eids: Vec> = self .attributes .iter() @@ -524,6 +531,34 @@ mod tests { ); } + #[test] + fn omits_empty_or_null_segments() { + let provider = provider(); + let headers = HeaderMap::new(); + + for segments in [json!([]), json!(null), json!("s1")] { + let body = json!({ "nonId": "li-example", "segments": segments }) + .to_string() + .into_bytes(); + let outcome = provider + .parse_response(ProviderResponse { + status: StatusCode::OK, + headers: &headers, + body: &body, + context: None, + }) + .expect("should parse response"); + let LookupOutcome::Found(eids) = outcome else { + panic!("should find the nonId EID"); + }; + + assert!( + eids[0].ext.is_none(), + "should omit segments {segments} from the nonId EID ext" + ); + } + } + #[test] fn maps_returned_attributes_to_sources_with_the_expires_header() { let provider = provider(); From d58593b43d4996cc4f11493e026adb2af37f660d Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 18:50:26 +0530 Subject: [PATCH 30/72] Read the first non-empty value among duplicate vendor cookies Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/provider.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs index 641206301..815b99ec8 100644 --- a/crates/trusted-server-core/src/ec/provider.rs +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -160,17 +160,20 @@ impl<'a> RequestCookies<'a> { Self(header) } - /// Returns the first value of cookie `name`, undecoded and unquoted. + /// Returns the first non-empty value of cookie `name`, undecoded and + /// unquoted. /// - /// An empty value reads as absent. + /// Empty values are skipped, so an empty duplicate (another path or + /// domain) does not hide a later value; a name with only empty values + /// reads as absent. #[must_use] pub fn get(&self, name: &str) -> Option<&'a str> { self.0 .split(';') .filter_map(|pair| pair.split_once('=')) - .find(|(key, _)| key.trim() == name) + .filter(|(key, _)| key.trim() == name) .map(|(_, value)| value.trim().trim_matches('"')) - .filter(|value| !value.is_empty()) + .find(|value| !value.is_empty()) } } @@ -978,6 +981,11 @@ mod tests { ); assert_eq!(cookies.get("a"), Some("1"), "should find leading cookie"); assert_eq!(cookies.get("empty"), None, "should ignore empty values"); + assert_eq!( + RequestCookies::new("vendor_id=; vendor_id=v-2").get("vendor_id"), + Some("v-2"), + "should skip an empty duplicate before a later value" + ); assert_eq!(cookies.get("missing"), None, "should miss absent cookies"); } From 567dec8bcaca46a45bdb256bfa292696c6f3596d Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 18:51:21 +0530 Subject: [PATCH 31/72] Check the lookup rate limit only once a request is built Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 77 +++++++++++++++++-- 1 file changed, 72 insertions(+), 5 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 892b400b9..e468d3f6b 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -588,10 +588,8 @@ impl IdentityCoordinator for ProviderCoordinator

{ continue; } } - if Self::rate_limited(registered, lookup) { - pending.early.push(skip(SkipReason::RateLimited)); - continue; - } + // The request is built before the rate limit is checked, so a + // provider that makes no lookup spends none of its budget. let mut request = match registered.provider.build_request(&input) { Ok(request) => request, Err(report) => { @@ -607,6 +605,10 @@ impl IdentityCoordinator for ProviderCoordinator

{ continue; } }; + if Self::rate_limited(registered, lookup) { + pending.early.push(skip(SkipReason::RateLimited)); + continue; + } if !destination_allowed(&request.url, ®istered.policy.allowed_domains) { pending.early.push(skip(SkipReason::Destination)); continue; @@ -738,7 +740,7 @@ mod tests { use crate::error::TrustedServerError; use crate::settings::EcPartner; use std::borrow::Cow; - use std::cell::RefCell; + use std::cell::{Cell, RefCell}; use std::num::NonZeroUsize; const EC_ID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; @@ -1991,4 +1993,69 @@ mod tests { "should keep the held ID" ); } + + /// An echo provider that makes no lookup. + struct NoLookupProvider(EchoProvider); + + impl IdentityProvider for NoLookupProvider { + fn capabilities(&self) -> &ProviderCapabilities { + self.0.capabilities() + } + + fn build_request( + &self, + _lookup: &LookupInput<'_>, + ) -> Result> { + Err(Report::new(ProviderError::Request)) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + self.0.parse_response(response) + } + } + + /// A limiter that counts the checks it is asked for. + #[derive(Default)] + struct CountingLimiter(Cell); + + impl RateLimiter for CountingLimiter { + fn exceeded( + &self, + _key: &str, + _hourly_limit: u32, + ) -> Result> { + self.0.set(self.0.get() + 1); + Ok(false) + } + } + + #[test] + fn a_provider_making_no_lookup_spends_no_rate_limit() { + let coordinator = ProviderCoordinator::new( + vec![ + RegisteredProvider::new( + provider_config(""), + NoLookupProvider(echo_provider(SourceInput::Anonymous)), + ) + .with_rate_limit(NonZeroU32::new(1).expect("should be non-zero")), + ], + partners(), + Duration::from_millis(200), + ) + .expect("should build coordinator"); + let transport = CannedTransport::new(ok_reply); + let limiter = CountingLimiter::default(); + + let results = run_with(&coordinator, &transport, &missing(), &granting(), &limiter); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::Input), + "should skip a provider that builds no request" + ); + assert_eq!(limiter.0.get(), 0, "should not check the rate limit"); + } } From 81e99065d69bbccaeb80e30b6dc480d17ac632e5 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 18:53:19 +0530 Subject: [PATCH 32/72] Look up the due sources a vendor cookie did not fill Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 183 +++++++++++++++++- 1 file changed, 173 insertions(+), 10 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index e468d3f6b..da6ea9f8c 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -521,7 +521,7 @@ impl IdentityCoordinator for ProviderCoordinator

{ }); continue; } - let (due, input_blocked) = Self::due_sources(registered, entry, lookup.now); + let (mut due, input_blocked) = Self::due_sources(registered, entry, lookup.now); // The vendor's own first-party cookies are read on every // navigation, so an opt-out or a changed ID is noticed while the // sources still hold live UIDs. Their IDs stand in for a lookup. @@ -537,18 +537,26 @@ impl IdentityCoordinator for ProviderCoordinator

{ CookieRead::Found(eids) => { let found = self.validated(provider, eids, lookup.now); if matches!(found.status, LookupStatus::Found { .. }) { + // Only the sources the cookies filled are spared a + // lookup; the provider's other due sources are + // still looked up. + due.retain(|source| { + found.entries.iter().all(|(filled, _)| filled != source) + }); // A cookie repeating the held UIDs is not rewritten, // so its relative expiry does not cause a write on // every navigation. if due.is_empty() && holds_same_uids(entry, &found, lookup.now) { pending.early.push(skip(SkipReason::NotDue)); - } else { - pending.from_cookies.push(Tagged { - provider, - value: found, - }); + continue; + } + pending.from_cookies.push(Tagged { + provider, + value: found, + }); + if due.is_empty() { + continue; } - continue; } } CookieRead::Absent => {} @@ -661,15 +669,33 @@ impl IdentityCoordinator for ProviderCoordinator

{ for Tagged { provider, value } in early { slots[provider.get()] = Some(self.lookup(provider, value, Vec::new())); } - for Tagged { provider, value } in generated.into_iter().chain(from_cookies) { - slots[provider.get()] = Some(value); + // A provider's cookie result replaces its early skip, and its later + // lookup or generated result is merged into the cookie result. + let mut filled: Vec> = + (0..self.providers.len()).map(|_| None).collect(); + for Tagged { provider, value } in from_cookies.into_iter().chain(generated) { + let slot = &mut filled[provider.get()]; + *slot = Some(match slot.take() { + Some(earlier) => merge_lookups(earlier, value), + None => value, + }); } for Tagged { provider, value } in responses { let context = contexts .iter() .find(|tagged| tagged.provider == provider) .map(|tagged| tagged.value.as_str()); - slots[provider.get()] = Some(self.interpret(provider, value, context, now)); + let later = self.interpret(provider, value, context, now); + let slot = &mut filled[provider.get()]; + *slot = Some(match slot.take() { + Some(earlier) => merge_lookups(earlier, later), + None => later, + }); + } + for (slot, value) in slots.iter_mut().zip(filled) { + if value.is_some() { + *slot = value; + } } let lookups: Vec = slots.into_iter().flatten().collect(); for lookup in &lookups { @@ -679,6 +705,45 @@ impl IdentityCoordinator for ProviderCoordinator

{ } } +/// Merges a provider's cookie result with its later lookup or generated +/// result for the sources the cookies did not fill. +/// +/// An opt-out wins; otherwise the found entries of both are kept, and a +/// later result that found nothing leaves the cookie result. +fn merge_lookups(earlier: ProviderLookup, later: ProviderLookup) -> ProviderLookup { + let LookupStatus::Found { dropped, .. } = later.status else { + return if later.status == LookupStatus::Skipped(SkipReason::OptedOut) + || !matches!(earlier.status, LookupStatus::Found { .. }) + { + later + } else { + earlier + }; + }; + if !matches!(earlier.status, LookupStatus::Found { .. }) { + return later; + } + let mut entries = later.entries; + for (source, eids) in earlier.entries { + if entries.iter().all(|(held, _)| *held != source) { + entries.push((source, eids)); + } + } + let merged = entries + .iter() + .flat_map(|(_, eids)| eids) + .map(|eid| eid.uids().len()) + .sum::(); + ProviderLookup { + status: LookupStatus::Found { + kept: u16::try_from(merged).unwrap_or(u16::MAX), + dropped, + }, + entries, + ..later + } +} + /// Whether the row already holds exactly the UIDs `found` read from cookies, /// for every source `found` returned. fn holds_same_uids(entry: Option<&KvEntry>, found: &ProviderLookup, now: UnixSeconds) -> bool { @@ -2058,4 +2123,102 @@ mod tests { ); assert_eq!(limiter.0.get(), 0, "should not check the rate limit"); } + + const SECOND_SOURCE: &str = "ids-b.example.com"; + + /// A two-source provider: its cookie fills the first source and its + /// lookup the second. + struct TwoSourceProvider(EchoProvider); + + impl IdentityProvider for TwoSourceProvider { + fn capabilities(&self) -> &ProviderCapabilities { + self.0.capabilities() + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + self.0.build_request(lookup) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + let mut outcome = self.0.parse_response(response)?; + if let LookupOutcome::Found(eids) = &mut outcome { + for eid in eids { + eid.source = Cow::Borrowed(SECOND_SOURCE); + } + } + Ok(outcome) + } + + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> CookieRead<'cookie> { + self.0.read_request_cookies(cookies) + } + } + + #[test] + fn a_cookie_spares_only_the_sources_it_fills() { + let mut provider = cookie_provider(SourceInput::Anonymous, Some("vendor_id")); + provider.0.sources = Box::new([ + SourceCapability { + source: SourceDomain::try_from("ids.example.com").expect("should parse"), + input: SourceInput::Anonymous, + }, + SourceCapability { + source: SourceDomain::try_from(SECOND_SOURCE).expect("should parse"), + input: SourceInput::Anonymous, + }, + ]); + let config: IdentityProviderConfig = toml::from_str(&format!( + "id = \"example-provider\"\nkind = \"echo\"\nendpoint = \"https://ids.example.com/lookup\"\nallowed_domains = [\"ids.example.com\"]\nsources = [\"ids.example.com\", \"{SECOND_SOURCE}\"]\n" + )) + .expect("should parse config"); + let partners: Vec = ["ids.example.com", SECOND_SOURCE] + .into_iter() + .map(|source| { + serde_json::from_value(serde_json::json!({ + "name": source, "source_domain": source, "bidstream_enabled": true + })) + .expect("should build partner") + }) + .collect(); + let coordinator = ProviderCoordinator::new( + vec![RegisteredProvider::new(config, TwoSourceProvider(provider))], + Arc::new(PartnerRegistry::from_config(&partners).expect("should build registry")), + Duration::from_millis(200), + ) + .expect("should build coordinator"); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &missing(), &granting()); + + assert_eq!( + *transport.sent.borrow(), + ["https://ids.example.com/lookup?due=1"], + "should look up only the source the cookie did not fill" + ); + let lookup = results.iter().next().expect("should report the provider"); + assert!( + matches!(lookup.status, LookupStatus::Found { kept: 2, .. }), + "should keep the cookie and lookup UIDs" + ); + let mut sources: Vec<&str> = lookup + .entries + .iter() + .map(|(source, _)| source.as_str()) + .collect(); + sources.sort_unstable(); + assert_eq!( + sources, + ["ids-b.example.com", "ids.example.com"], + "should store both sources" + ); + } } From bfbe28b4629f74cf492f8db3c40c0da730b56f67 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 18:54:34 +0530 Subject: [PATCH 33/72] Skip UID2 where GDPR applies Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 65 ++++++++++++++++++- crates/trusted-server-core/src/ec/provider.rs | 13 ++++ .../src/uid2.rs | 18 +++++ trusted-server.example.toml | 3 +- 4 files changed, 97 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index da6ea9f8c..a26458807 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -496,7 +496,9 @@ impl IdentityCoordinator for ProviderCoordinator

{ pending.early.push(skip(SkipReason::NoRow)); continue; } - if !consented || vendor_denied { + let region_excluded = + lookup.consent.gdpr_applies && !registered.provider.runs_where_gdpr_applies(); + if !consented || vendor_denied || region_excluded { pending.early.push(skip(SkipReason::Consent)); continue; } @@ -1389,6 +1391,67 @@ mod tests { ); } + /// An echo provider whose IDs are not offered where GDPR applies. + struct OutsideGdprProvider(EchoProvider); + + impl IdentityProvider for OutsideGdprProvider { + fn capabilities(&self) -> &ProviderCapabilities { + self.0.capabilities() + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + self.0.build_request(lookup) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + self.0.parse_response(response) + } + + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> CookieRead<'cookie> { + self.0.read_request_cookies(cookies) + } + + fn runs_where_gdpr_applies(&self) -> bool { + false + } + } + + #[test] + fn a_provider_outside_gdpr_regions_is_skipped_where_gdpr_applies() { + let coordinator = ProviderCoordinator::new( + vec![RegisteredProvider::new( + provider_config(""), + OutsideGdprProvider(cookie_provider(SourceInput::Anonymous, Some("vendor_id"))), + )], + partners(), + Duration::from_millis(200), + ) + .expect("should build coordinator"); + let transport = CannedTransport::new(ok_reply); + + let gdpr = run(&coordinator, &transport, &missing(), &tcf_consent(vec![])); + let outside = run(&coordinator, &transport, &missing(), &granting()); + + assert_eq!( + only_status(&gdpr), + LookupStatus::Skipped(SkipReason::Consent), + "should neither read cookies nor look up where GDPR applies" + ); + assert!( + matches!(only_status(&outside), LookupStatus::Found { .. }), + "should read the cookie where GDPR does not apply" + ); + } + #[test] fn hashed_email_sources_are_skipped_for_input() { let coordinator = coordinator_with(RegisteredProvider::new( diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs index 815b99ec8..c02e1d4f0 100644 --- a/crates/trusted-server-core/src/ec/provider.rs +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -87,6 +87,15 @@ pub trait IdentityProvider: Send + Sync { let _ = cookies; CookieRead::Absent } + + /// Whether this provider runs for visitors where GDPR applies. + /// + /// A provider whose IDs are not offered in GDPR regions returns `false`, + /// and the coordinator then skips it there for consent: its cookies are + /// not read and no lookup is made. The default runs everywhere. + fn runs_where_gdpr_applies(&self) -> bool { + true + } } impl IdentityProvider for Box

{ @@ -118,6 +127,10 @@ impl IdentityProvider for Box

{ ) -> CookieRead<'cookie> { (**self).read_request_cookies(cookies) } + + fn runs_where_gdpr_applies(&self) -> bool { + (**self).runs_where_gdpr_applies() + } } /// What a provider found in the vendor's own first-party cookies. diff --git a/crates/trusted-server-identity-providers/src/uid2.rs b/crates/trusted-server-identity-providers/src/uid2.rs index f025113f0..01190b2a6 100644 --- a/crates/trusted-server-identity-providers/src/uid2.rs +++ b/crates/trusted-server-identity-providers/src/uid2.rs @@ -31,6 +31,9 @@ //! A `latestToken` of `"optout"` in any of those cookies reports an opt-out: //! no lookup is made and stored UID2 UIDs are removed. //! +//! UID2 is not offered where GDPR applies: there the provider is skipped +//! for consent, so its cookies are not read and no refresh is made. +//! //! Parsing is lenient: an undecodable cookie, a missing token or expiry, //! or a blank or opt-out token reads as no ID. A JSON value is never stored //! as a token. @@ -276,6 +279,11 @@ impl IdentityProvider for Uid2Provider { &self.capabilities } + /// UID2 is not offered where GDPR applies, so it is skipped there. + fn runs_where_gdpr_applies(&self) -> bool { + false + } + fn build_request( &self, lookup: &LookupInput<'_>, @@ -420,6 +428,16 @@ mod tests { format!("__uid_2={}", urlencoding::encode(&identity.to_string())) } + #[test] + fn is_not_run_where_gdpr_applies() { + let provider = provider(&json!({})); + + assert!( + !provider.runs_where_gdpr_applies(), + "should not run UID2 where GDPR applies" + ); + } + #[test] fn reads_only_the_advertising_token_until_identity_expiry() { let header = identity_cookie(&json!({ diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 609f185d9..4b7dc375e 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -259,7 +259,8 @@ pull_sync_concurrency = 3 # refreshed server side from `refresh_from`, with no API key and no email. # Tokens are never generated: that needs an email or phone number. Without # `endpoint`, the refresh token is never stored. The authenticated -# `/_ts/admin/ec/{id}` view shows refresh state as "[redacted]". +# `/_ts/admin/ec/{id}` view shows refresh state as "[redacted]". UID2 is not +# offered where GDPR applies, so it is skipped there. # [[ec.identity_providers]] # id = "uid2" # kind = "uid2" From 8b41fde5a3b23e356fe43e63b8784b81fda1bf1e Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 18:54:46 +0530 Subject: [PATCH 34/72] Show the TCF vendor ID for the Criteo and Unified ID examples Signed-off-by: dhruv8sh --- trusted-server.example.toml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 4b7dc375e..cd682cedf 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -135,7 +135,10 @@ pull_sync_concurrency = 3 # must be claimed by only one enabled provider. # sources = ["ids.example.com"] # Optional: require TCF vendor consent for lookups and for sending any ID of -# these sources to bidders, including browser IDs, even while disabled. +# these sources to bidders, including browser IDs, even while disabled. Set it +# to the vendor's IAB Global Vendor List ID for every kind whose vendor is +# registered there; without it, GDPR traffic is gated only by the purpose +# consents. # tcf_vendor_id = 9999 # "page" (default) collects before the response; "after_response" collects # after it is sent, only for an EC issued on an earlier request. Keep @@ -281,7 +284,7 @@ pull_sync_concurrency = 3 # `_pubcid_optout` cookie suppresses it. Takes exactly one source. When no # cookie holds an ID and none is stored, Trusted Server generates a random # UUID and stores it in the row only (no cookie is set); a later Prebid cookie -# replaces it. +# replaces it. SharedID is the publisher's own ID, so it has no TCF vendor. # [[ec.identity_providers]] # id = "sharedid" # kind = "sharedid" @@ -298,6 +301,7 @@ pull_sync_concurrency = 3 # kind = "criteo" # enabled = false # sources = ["criteo.example.com"] +# tcf_vendor_id = 9999 # [ec.identity_providers.settings] # cookie_names = ["cto_bidid"] # @@ -311,6 +315,7 @@ pull_sync_concurrency = 3 # kind = "unifiedid" # enabled = false # sources = ["unifiedid.example.com"] +# tcf_vendor_id = 9999 # [ec.identity_providers.settings] # cookie_names = ["pbjs-unifiedid"] From f910910e46d6a550d2c90eb3640d8d9d17f8372b Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 18:54:52 +0530 Subject: [PATCH 35/72] Note that the ConnectID opt-out is not visible server side Signed-off-by: dhruv8sh --- crates/trusted-server-identity-providers/src/connectid.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/trusted-server-identity-providers/src/connectid.rs b/crates/trusted-server-identity-providers/src/connectid.rs index c6823a8c9..a6fd31c3e 100644 --- a/crates/trusted-server-identity-providers/src/connectid.rs +++ b/crates/trusted-server-identity-providers/src/connectid.rs @@ -8,6 +8,10 @@ //! drops it once that time has passed. A missing `ttl` uses the shared //! default TTL. No server call is made. //! +//! Yahoo's opt-out (`connectIdOptOut` = `1`) is kept in local storage, which +//! a server never sees, so it is not honoured here: a `ConnectID` cookie left +//! behind after an opt-out is still read until it expires. +//! //! This kind has no Lockr capability: Lockr AIM is not documented to //! deliver `ConnectID`, so a `lockr` settings table is rejected. //! From 4324589280f39633910d499aa5d1fa790853ea93 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 5 Nov 2026 19:49:09 +0530 Subject: [PATCH 36/72] Drop UID2 IDs, including those LiveIntent resolves, where the visitor's jurisdiction is GDPR Providers now name the sources they exclude where GDPR applies. For a visitor whose detected jurisdiction is GDPR, those sources are neither looked up nor stored, and the UIDs the row holds for them are removed. A TCF string alone no longer counts as GDPR applying. Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 188 ++++++++++++++++-- .../src/ec/coordinator/results.rs | 4 + crates/trusted-server-core/src/ec/provider.rs | 19 +- .../src/liveintent.rs | 27 ++- .../src/uid2.rs | 10 +- trusted-server.example.toml | 5 +- 6 files changed, 225 insertions(+), 28 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index a26458807..1302cf1bc 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -21,6 +21,7 @@ use self::transport::{ }; use self::validate::{ValidatedEntries, ValidationPolicy, validate}; use crate::consent::ConsentContext; +use crate::consent::jurisdiction::Jurisdiction; use crate::consent::{allows_eid_persistence, tcf_vendor_consent_denied}; use crate::ec::EcKvSnapshot; use crate::ec::consent::ec_consent_granted; @@ -102,6 +103,8 @@ pub struct PendingLookups { early: Vec>, /// Providers whose IDs were generated at dispatch, validated. generated: Vec>, + /// Sources each provider may not collect or keep where GDPR applies. + excluded: Vec>>, /// Provider state to hand back with each in-flight request's response. contexts: Vec>, /// Providers whose IDs were read from first-party cookies, validated. @@ -117,6 +120,7 @@ impl Default for PendingLookups { now: UnixSeconds::new(0), early: Vec::new(), generated: Vec::new(), + excluded: Vec::new(), contexts: Vec::new(), from_cookies: Vec::new(), } @@ -313,6 +317,23 @@ impl ProviderCoordinator

{ (due, input_blocked) } + /// The provider's sources it may not collect or keep for this request: + /// those it excludes where GDPR applies, when the visitor's jurisdiction + /// is GDPR. A TCF string alone does not decide this, because some CMPs + /// set one for every visitor. + fn excluded_sources(registered: &Registered

, consent: &ConsentContext) -> Vec { + if !matches!(consent.jurisdiction, Jurisdiction::Gdpr) { + return Vec::new(); + } + registered + .policy + .sources + .iter() + .filter(|source| registered.provider.excluded_where_gdpr_applies(source)) + .cloned() + .collect() + } + /// Whether the provider's hourly limit is reached. Limiter errors fail /// open, matching pull sync. /// @@ -343,7 +364,10 @@ impl ProviderCoordinator

{ entries: ValidatedEntries, ) -> ProviderLookup { let registered = &self.providers[provider.get()]; - let cleared = if status == LookupStatus::Skipped(SkipReason::OptedOut) { + let cleared = if matches!( + status, + LookupStatus::Skipped(SkipReason::OptedOut | SkipReason::Region) + ) { registered.policy.sources.to_vec() } else { Vec::new() @@ -496,12 +520,21 @@ impl IdentityCoordinator for ProviderCoordinator

{ pending.early.push(skip(SkipReason::NoRow)); continue; } - let region_excluded = - lookup.consent.gdpr_applies && !registered.provider.runs_where_gdpr_applies(); - if !consented || vendor_denied || region_excluded { + if !consented || vendor_denied { pending.early.push(skip(SkipReason::Consent)); continue; } + let excluded = Self::excluded_sources(registered, lookup.consent); + if !excluded.is_empty() { + if excluded.len() == registered.policy.sources.len() { + pending.early.push(skip(SkipReason::Region)); + continue; + } + pending.excluded.push(Tagged { + provider, + value: excluded.clone(), + }); + } // A remembered opt-out keeps the provider from being called until // it lapses. It clears only sources the row still stores, so once // they are gone the after-response path skips its re-read. @@ -524,6 +557,7 @@ impl IdentityCoordinator for ProviderCoordinator

{ continue; } let (mut due, input_blocked) = Self::due_sources(registered, entry, lookup.now); + due.retain(|source| !excluded.contains(source)); // The vendor's own first-party cookies are read on every // navigation, so an opt-out or a changed ID is noticed while the // sources still hold live UIDs. Their IDs stand in for a lookup. @@ -659,6 +693,7 @@ impl IdentityCoordinator for ProviderCoordinator

{ now, early, generated, + excluded, contexts, from_cookies, } = pending; @@ -699,6 +734,11 @@ impl IdentityCoordinator for ProviderCoordinator

{ *slot = value; } } + for Tagged { provider, value } in excluded { + if let Some(lookup) = slots[provider.get()].as_mut() { + exclude_sources(lookup, value); + } + } let lookups: Vec = slots.into_iter().flatten().collect(); for lookup in &lookups { log_outcome(lookup); @@ -707,6 +747,21 @@ impl IdentityCoordinator for ProviderCoordinator

{ } } +/// Drops `sources` from a provider's result and removes them from the row. +fn exclude_sources(lookup: &mut ProviderLookup, sources: Vec) { + lookup + .entries + .retain(|(source, _)| !sources.contains(source)); + if matches!(lookup.status, LookupStatus::Found { .. }) && lookup.entries.is_empty() { + lookup.status = LookupStatus::NoMatch; + } + for source in sources { + if !lookup.cleared.contains(&source) { + lookup.cleared.push(source); + } + } +} + /// Merges a provider's cookie result with its later lookup or generated /// result for the sources the cookies did not fill. /// @@ -795,7 +850,6 @@ fn log_outcome(lookup: &ProviderLookup) { #[cfg(test)] mod tests { use super::*; - use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::TcfConsent; use crate::ec::eid_types::{AgentType, EidFields, Expiry, UidValue}; use crate::ec::kv::KvIdentityGraph; @@ -1420,8 +1474,8 @@ mod tests { self.0.read_request_cookies(cookies) } - fn runs_where_gdpr_applies(&self) -> bool { - false + fn excluded_where_gdpr_applies(&self, _source: &SourceDomain) -> bool { + true } } @@ -1441,17 +1495,51 @@ mod tests { let gdpr = run(&coordinator, &transport, &missing(), &tcf_consent(vec![])); let outside = run(&coordinator, &transport, &missing(), &granting()); + let lookup = gdpr.iter().next().expect("should report the provider"); assert_eq!( - only_status(&gdpr), - LookupStatus::Skipped(SkipReason::Consent), + lookup.status, + LookupStatus::Skipped(SkipReason::Region), "should neither read cookies nor look up where GDPR applies" ); + assert_eq!( + lookup.cleared, + [SourceDomain::try_from("ids.example.com").expect("should parse")], + "should remove the stored UIDs where GDPR applies" + ); assert!( matches!(only_status(&outside), LookupStatus::Found { .. }), "should read the cookie where GDPR does not apply" ); } + #[test] + fn a_tcf_string_alone_does_not_exclude_a_provider_outside_gdpr() { + // Arrange: a CMP that sets a TCF string for a US visitor. + let coordinator = ProviderCoordinator::new( + vec![RegisteredProvider::new( + provider_config(""), + OutsideGdprProvider(cookie_provider(SourceInput::Anonymous, Some("vendor_id"))), + )], + partners(), + Duration::from_millis(200), + ) + .expect("should build coordinator"); + let transport = CannedTransport::new(ok_reply); + let consent = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + ..tcf_consent(vec![]) + }; + + // Act + let results = run(&coordinator, &transport, &missing(), &consent); + + // Assert + assert!( + matches!(only_status(&results), LookupStatus::Found { .. }), + "should collect where only a TCF string is present" + ); + } + #[test] fn hashed_email_sources_are_skipped_for_input() { let coordinator = coordinator_with(RegisteredProvider::new( @@ -2226,8 +2314,9 @@ mod tests { } } - #[test] - fn a_cookie_spares_only_the_sources_it_fills() { + fn two_source_coordinator( + wrap: impl FnOnce(TwoSourceProvider) -> P, + ) -> ProviderCoordinator

{ let mut provider = cookie_provider(SourceInput::Anonymous, Some("vendor_id")); provider.0.sources = Box::new([ SourceCapability { @@ -2252,12 +2341,20 @@ mod tests { .expect("should build partner") }) .collect(); - let coordinator = ProviderCoordinator::new( - vec![RegisteredProvider::new(config, TwoSourceProvider(provider))], + ProviderCoordinator::new( + vec![RegisteredProvider::new( + config, + wrap(TwoSourceProvider(provider)), + )], Arc::new(PartnerRegistry::from_config(&partners).expect("should build registry")), Duration::from_millis(200), ) - .expect("should build coordinator"); + .expect("should build coordinator") + } + + #[test] + fn a_cookie_spares_only_the_sources_it_fills() { + let coordinator = two_source_coordinator(|provider| provider); let transport = CannedTransport::new(ok_reply); let results = run(&coordinator, &transport, &missing(), &granting()); @@ -2284,4 +2381,67 @@ mod tests { "should store both sources" ); } + + /// A two-source provider whose cookie source is excluded where GDPR + /// applies. + struct CookieSourceExcluded(TwoSourceProvider); + + impl IdentityProvider for CookieSourceExcluded { + fn capabilities(&self) -> &ProviderCapabilities { + self.0.capabilities() + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + self.0.build_request(lookup) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + self.0.parse_response(response) + } + + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> CookieRead<'cookie> { + self.0.read_request_cookies(cookies) + } + + fn excluded_where_gdpr_applies(&self, source: &SourceDomain) -> bool { + source.as_str() == "ids.example.com" + } + } + + #[test] + fn an_excluded_source_is_neither_collected_nor_kept_where_gdpr_applies() { + // Arrange + let coordinator = two_source_coordinator(CookieSourceExcluded); + let transport = CannedTransport::new(ok_reply); + + // Act + let results = run(&coordinator, &transport, &missing(), &tcf_consent(vec![])); + + // Assert + let lookup = results.iter().next().expect("should report the provider"); + let sources: Vec<&str> = lookup + .entries + .iter() + .map(|(source, _)| source.as_str()) + .collect(); + assert_eq!( + sources, + [SECOND_SOURCE], + "should keep only the source that is not excluded" + ); + assert_eq!( + lookup.cleared, + [SourceDomain::try_from("ids.example.com").expect("should parse")], + "should remove the excluded source from the row" + ); + } } diff --git a/crates/trusted-server-core/src/ec/coordinator/results.rs b/crates/trusted-server-core/src/ec/coordinator/results.rs index f52f211c2..6a3144bfe 100644 --- a/crates/trusted-server-core/src/ec/coordinator/results.rs +++ b/crates/trusted-server-core/src/ec/coordinator/results.rs @@ -66,6 +66,9 @@ pub enum SkipReason { /// The vendor or its cookies report an opt-out; the provider's stored /// UIDs are removed. OptedOut, + /// Every source of the provider is excluded where GDPR applies; the + /// provider's stored UIDs are removed. + Region, } /// One outcome per provider per navigation. @@ -111,6 +114,7 @@ impl LookupStatus { Self::Skipped(SkipReason::NotDue) => "skipped_not_due", Self::Skipped(SkipReason::Input) => "skipped_input", Self::Skipped(SkipReason::RateLimited) => "skipped_rate_limited", + Self::Skipped(SkipReason::Region) => "skipped_region", Self::Skipped(SkipReason::Destination) => "destination_rejected", Self::Skipped(SkipReason::OptedOut) => "skipped_opted_out", } diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs index c02e1d4f0..ae70025ac 100644 --- a/crates/trusted-server-core/src/ec/provider.rs +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -88,13 +88,16 @@ pub trait IdentityProvider: Send + Sync { CookieRead::Absent } - /// Whether this provider runs for visitors where GDPR applies. + /// Whether `source`'s IDs must not be collected or kept where GDPR applies. /// - /// A provider whose IDs are not offered in GDPR regions returns `false`, - /// and the coordinator then skips it there for consent: its cookies are - /// not read and no lookup is made. The default runs everywhere. - fn runs_where_gdpr_applies(&self) -> bool { - true + /// Where the visitor's jurisdiction is GDPR, the coordinator neither looks + /// up nor stores an excluded source and removes the UIDs the row holds for + /// it. A provider whose every source is excluded is skipped entirely: its + /// cookies are not read and no lookup is made. The default excludes + /// nothing. + fn excluded_where_gdpr_applies(&self, source: &SourceDomain) -> bool { + let _ = source; + false } } @@ -128,8 +131,8 @@ impl IdentityProvider for Box

{ (**self).read_request_cookies(cookies) } - fn runs_where_gdpr_applies(&self) -> bool { - (**self).runs_where_gdpr_applies() + fn excluded_where_gdpr_applies(&self, source: &SourceDomain) -> bool { + (**self).excluded_where_gdpr_applies(source) } } diff --git a/crates/trusted-server-identity-providers/src/liveintent.rs b/crates/trusted-server-identity-providers/src/liveintent.rs index 2579e9db0..c37e6ef10 100644 --- a/crates/trusted-server-identity-providers/src/liveintent.rs +++ b/crates/trusted-server-identity-providers/src/liveintent.rs @@ -55,10 +55,13 @@ pub const ATTRIBUTE_SOURCES: &[(&str, &str)] = &[ ("zetassp", "zeta-ssp.liveintent.com"), ("vidazoo", "liveintent.vidazoo.com"), ("nexxen", "liveintent.unrulymedia.com"), - ("uid2", "uidapi.com"), + ("uid2", UID2_SOURCE), (TDID_ATTRIBUTE, "adserver.org"), ]; +/// The source of UID2 tokens `LiveIntent` resolves. +const UID2_SOURCE: &str = "uidapi.com"; + /// The attribute holding the Trade Desk ID `LiveIntent` resolved. const TDID_ATTRIBUTE: &str = "thetradedesk"; @@ -254,6 +257,12 @@ impl IdentityProvider for LiveIntentProvider { &self.capabilities } + /// UID2 tokens `LiveIntent` resolves are not collected or kept where GDPR + /// applies, matching the `uid2` provider. + fn excluded_where_gdpr_applies(&self, source: &SourceDomain) -> bool { + source.as_str() == UID2_SOURCE + } + fn build_request( &self, lookup: &LookupInput<'_>, @@ -430,6 +439,22 @@ mod tests { } } + #[test] + fn excludes_only_uid2_where_gdpr_applies() { + let provider = provider(); + let uid2 = SourceDomain::try_from("uidapi.com").expect("should build source"); + let nonid = SourceDomain::try_from("liveintent.com").expect("should build source"); + + assert!( + provider.excluded_where_gdpr_applies(&uid2), + "should exclude UID2 tokens where GDPR applies" + ); + assert!( + !provider.excluded_where_gdpr_applies(&nonid), + "should keep LiveIntent's own ID where GDPR applies" + ); + } + #[test] fn sends_the_first_present_first_party_cookie_as_duid() { let provider = provider(); diff --git a/crates/trusted-server-identity-providers/src/uid2.rs b/crates/trusted-server-identity-providers/src/uid2.rs index 01190b2a6..a8cf14ead 100644 --- a/crates/trusted-server-identity-providers/src/uid2.rs +++ b/crates/trusted-server-identity-providers/src/uid2.rs @@ -279,9 +279,10 @@ impl IdentityProvider for Uid2Provider { &self.capabilities } - /// UID2 is not offered where GDPR applies, so it is skipped there. - fn runs_where_gdpr_applies(&self) -> bool { - false + /// UID2 is not offered where GDPR applies, so none of its sources is + /// collected or kept there. + fn excluded_where_gdpr_applies(&self, _source: &SourceDomain) -> bool { + true } fn build_request( @@ -431,9 +432,10 @@ mod tests { #[test] fn is_not_run_where_gdpr_applies() { let provider = provider(&json!({})); + let source = SourceDomain::try_from("uid2.example.com").expect("should build source"); assert!( - !provider.runs_where_gdpr_applies(), + provider.excluded_where_gdpr_applies(&source), "should not run UID2 where GDPR applies" ); } diff --git a/trusted-server.example.toml b/trusted-server.example.toml index cd682cedf..255b3a1e6 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -192,6 +192,7 @@ pull_sync_concurrency = 3 # liveintent.sonobi.com, triplelift liveintent.triplelift.com, zetassp # zeta-ssp.liveintent.com, vidazoo liveintent.vidazoo.com, nexxen # liveintent.unrulymedia.com, uid2 uidapi.com, thetradedesk adserver.org. +# Where GDPR applies, uid2 is neither resolved nor kept, as for kind "uid2". # `nonId` is LiveIntent's own ID and carries the reply's `segments` as EID # `ext`; the others are partner IDs, sent with UID `ext.provider` naming # LiveIntent (the Trade Desk ID also with `rtiPartner` and EID `matcher`). @@ -263,7 +264,9 @@ pull_sync_concurrency = 3 # Tokens are never generated: that needs an email or phone number. Without # `endpoint`, the refresh token is never stored. The authenticated # `/_ts/admin/ec/{id}` view shows refresh state as "[redacted]". UID2 is not -# offered where GDPR applies, so it is skipped there. +# offered where GDPR applies: for a visitor whose detected jurisdiction is +# GDPR it is skipped and the UIDs the row holds for it are removed. A TCF +# string alone does not count, since some CMPs set one for every visitor. # [[ec.identity_providers]] # id = "uid2" # kind = "uid2" From 7665a01affd3a23104c97770c4ecde967925eb00 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 17:16:08 +0530 Subject: [PATCH 37/72] Generate a SharedID only when the publisher opts in Signed-off-by: dhruv8sh --- .../src/sharedid.rs | 49 +++++++++++++++---- trusted-server.example.toml | 11 +++-- 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/crates/trusted-server-identity-providers/src/sharedid.rs b/crates/trusted-server-identity-providers/src/sharedid.rs index b9c08dd3f..cba5f75fd 100644 --- a/crates/trusted-server-identity-providers/src/sharedid.rs +++ b/crates/trusted-server-identity-providers/src/sharedid.rs @@ -7,11 +7,14 @@ //! opt-out, which removes a stored `SharedID`. //! No server call is made. //! -//! `SharedID` is a publisher first-party ID with no vendor server, so when -//! the cookies hold none and no live `SharedID` is stored, Trusted Server -//! generates one itself: a random UUID v4, stored in the row only. No cookie -//! is set, so Prebid's own cookie is never touched; when Prebid later writes -//! one, the cookie's ID replaces the generated one. +//! `SharedID` is a publisher first-party ID with no vendor server. With +//! `generate = true`, when the cookies hold none and no live `SharedID` is +//! stored, Trusted Server generates one itself: a random UUID v4, stored in +//! the row only. No cookie is set, so Prebid's own cookie is never touched; +//! when Prebid later writes one, the cookie's ID replaces the generated one. +//! Generation is off by default: Prebid may keep its opt-out in localStorage, +//! which the server cannot see, so only a publisher who knows its opt-out +//! lives in the `_pubcid_optout` cookie should turn it on. use std::borrow::Cow; @@ -42,6 +45,10 @@ pub struct SharedIdSettings { /// Cookies holding the `SharedID`, read in order. #[serde(default = "SharedIdSettings::default_cookie_names")] pub cookie_names: Vec, + /// Whether to generate a `SharedID` when the cookies hold none. Off by + /// default, because a localStorage opt-out is invisible to the server. + #[serde(default)] + pub generate: bool, } impl SharedIdSettings { @@ -56,6 +63,7 @@ pub struct SharedIdProvider { capabilities: ProviderCapabilities, source: SourceDomain, cookies: CookieReader, + generate: bool, } impl ProviderKind for SharedId { @@ -76,6 +84,7 @@ impl ProviderKind for SharedId { capabilities, source, cookies: CookieReader::own(common, settings.cookie_names), + generate: settings.generate, }) } } @@ -120,6 +129,9 @@ impl IdentityProvider for SharedIdProvider { } fn generate(&self, _lookup: &LookupInput<'_>) -> Option>> { + if !self.generate { + return None; + } Some(vec![device_eid( &self.source, Cow::Owned(Uuid::new_v4().to_string()), @@ -139,10 +151,15 @@ mod tests { use super::*; fn provider() -> Box { + provider_with(&json!({})) + } + + fn provider_with(settings: &serde_json::Value) -> Box { let config: IdentityProviderConfig = serde_json::from_value(json!({ "id": "sharedid", "kind": "sharedid", "sources": ["sharedid.example.com"], + "settings": settings, })) .expect("should parse provider config"); build_kind::(&config, &ProviderSecrets::default()).expect("should build provider") @@ -191,8 +208,7 @@ mod tests { } } - #[test] - fn generates_a_random_uuid_when_none_is_held() { + fn generate_with(provider: &dyn IdentityProvider) -> Option>> { let consent = trusted_server_core::consent::ConsentContext::default(); let lookup = LookupInput { ec_id: "ec-example", @@ -207,10 +223,23 @@ mod tests { cookies: RequestCookies::default(), hashed_email: None, }; - let provider = provider(); + provider.generate(&lookup) + } + + #[test] + fn generates_nothing_by_default() { + assert!( + generate_with(&*provider()).is_none(), + "should not generate without generate = true" + ); + } + + #[test] + fn generates_a_random_uuid_when_none_is_held() { + let provider = provider_with(&json!({ "generate": true })); - let first = provider.generate(&lookup).expect("should generate an ID"); - let second = provider.generate(&lookup).expect("should generate an ID"); + let first = generate_with(&*provider).expect("should generate an ID"); + let second = generate_with(&*provider).expect("should generate an ID"); let uid = &first[0].uids[0]; assert_eq!( diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 255b3a1e6..0c374a371 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -284,10 +284,12 @@ pull_sync_concurrency = 3 # Prebid SharedID (kind "sharedid"): no lookup and no endpoint. Reads the # publisher common ID from Prebid's first-party `_sharedid` cookie, falling # back to the legacy `_pubcid`, as a device-based (atype 1) ID. The -# `_pubcid_optout` cookie suppresses it. Takes exactly one source. When no -# cookie holds an ID and none is stored, Trusted Server generates a random -# UUID and stores it in the row only (no cookie is set); a later Prebid cookie -# replaces it. SharedID is the publisher's own ID, so it has no TCF vendor. +# `_pubcid_optout` cookie suppresses it. Takes exactly one source. With +# generate = true, when no cookie holds an ID and none is stored, Trusted Server +# generates a random UUID and stores it in the row only (no cookie is set); a +# later Prebid cookie replaces it. Leave it off unless Prebid keeps its opt-out +# in the `_pubcid_optout` cookie: a localStorage opt-out is invisible to the +# server. SharedID is the publisher's own ID, so it has no TCF vendor. # [[ec.identity_providers]] # id = "sharedid" # kind = "sharedid" @@ -295,6 +297,7 @@ pull_sync_concurrency = 3 # sources = ["sharedid.example.com"] # [ec.identity_providers.settings] # cookie_names = ["_sharedid", "_pubcid"] +# generate = false # # Criteo (kind "criteo"): no lookup and no endpoint. Reads Criteo's bid ID # from the first-party `cto_bidid` cookie Prebid's Criteo module keeps and From 40514968872672330dbb793f09fe7417c90f537b Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 17:18:47 +0530 Subject: [PATCH 38/72] Check every duplicate cookie for opt-outs and skip expired cookie IDs Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/provider.rs | 31 +++++- .../src/common.rs | 19 +++- .../src/connectid.rs | 47 ++++++--- .../src/uid2.rs | 98 ++++++++++++++----- 4 files changed, 150 insertions(+), 45 deletions(-) diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs index ae70025ac..22ae55918 100644 --- a/crates/trusted-server-core/src/ec/provider.rs +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -184,12 +184,23 @@ impl<'a> RequestCookies<'a> { /// reads as absent. #[must_use] pub fn get(&self, name: &str) -> Option<&'a str> { + self.get_all(name).next() + } + + /// Every non-empty value of the cookie `name`, in header order. + /// + /// A request can carry several cookies with one name (set on different + /// paths or domains); opt-out checks must look at all of them. + pub fn get_all<'name>(&self, name: &'name str) -> impl Iterator + 'name + where + 'a: 'name, + { self.0 .split(';') .filter_map(|pair| pair.split_once('=')) - .filter(|(key, _)| key.trim() == name) + .filter(move |(key, _)| key.trim() == name) .map(|(_, value)| value.trim().trim_matches('"')) - .find(|value| !value.is_empty()) + .filter(|value| !value.is_empty()) } } @@ -816,6 +827,22 @@ mod tests { ); } + #[test] + fn cookies_get_all_returns_every_non_empty_duplicate() { + let cookies = RequestCookies::new("id=; other=x; id=first; id=second"); + + assert_eq!( + cookies.get_all("id").collect::>(), + vec!["first", "second"], + "should return every non-empty value in order" + ); + assert_eq!( + cookies.get("id"), + Some("first"), + "should get the first value" + ); + } + #[test] fn request_debug_hides_query_and_headers() { let mut headers = http::HeaderMap::new(); diff --git a/crates/trusted-server-identity-providers/src/common.rs b/crates/trusted-server-identity-providers/src/common.rs index 21b38171a..9b3ca954f 100644 --- a/crates/trusted-server-identity-providers/src/common.rs +++ b/crates/trusted-server-identity-providers/src/common.rs @@ -8,6 +8,7 @@ use std::time::Duration; use error_stack::Report; use serde::Deserialize; use serde_json::value::RawValue; +use trusted_server_core::ec::current_timestamp; use trusted_server_core::ec::eid_types::SourceDomain; use trusted_server_core::ec::provider::{ EidLevelFields, EidShape, ProviderCapabilities, ProviderCommon, ProviderConfigError, @@ -298,12 +299,13 @@ impl CookieReader { self.own .iter() .chain(self.lockr.iter()) - .filter_map(|name| cookies.get(name)) + .flat_map(|name| cookies.get_all(name)) .any(matches) } - /// Returns the first value `parse` accepts, trying the own cookies - /// before the Lockr cookies. + /// Returns the first value `parse` accepts, trying every value of each + /// own cookie before the Lockr cookies, so an unusable or expired value + /// never hides a later usable one. pub(crate) fn read<'cookie, T>( &self, cookies: RequestCookies<'cookie>, @@ -312,13 +314,13 @@ impl CookieReader { if let Some(found) = self .own .iter() - .filter_map(|name| cookies.get(name)) + .flat_map(|name| cookies.get_all(name)) .find_map(&mut parse) { return Some(found); } let (name, found) = self.lockr.iter().find_map(|name| { - let found = parse(cookies.get(name)?)?; + let found = cookies.get_all(name).find_map(&mut parse)?; Some((name, found)) })?; log::debug!( @@ -329,6 +331,13 @@ impl CookieReader { } } +/// Whether a cookie's stated expiry has already passed. +/// +/// Cookie readers skip such a value so it never hides a later live one. +pub(crate) fn is_expired(expiry: &ProviderExpiry) -> bool { + matches!(expiry, ProviderExpiry::At(at) if at.get() <= current_timestamp()) +} + /// A complete `OpenRTB` EID as a vendor returns it, borrowed from the body. /// /// Unknown fields are ignored so one new vendor field does not fail the diff --git a/crates/trusted-server-identity-providers/src/connectid.rs b/crates/trusted-server-identity-providers/src/connectid.rs index a6fd31c3e..da12f8244 100644 --- a/crates/trusted-server-identity-providers/src/connectid.rs +++ b/crates/trusted-server-identity-providers/src/connectid.rs @@ -32,7 +32,8 @@ use trusted_server_core::ec::provider::{ }; use crate::common::{ - CookieReader, DEFAULT_TTL, cookie_capabilities, cookie_eid, is_usable_id, percent_decoded, + CookieReader, DEFAULT_TTL, cookie_capabilities, cookie_eid, is_expired, is_usable_id, + percent_decoded, }; /// Longest time a `ConnectID` is kept after its last sync. @@ -112,10 +113,9 @@ fn connect_id(value: &str) -> Option<(String, ProviderExpiry)> { } let ttl = cookie.ttl.map_or(DEFAULT_TTL, from_millis).min(MAX_TTL); let expires = from_millis(cookie.last_synced_ms).saturating_add(ttl); - Some(( - cookie.connect_id, - ProviderExpiry::At(UnixSeconds::new(expires.as_secs())), - )) + let expiry = ProviderExpiry::At(UnixSeconds::new(expires.as_secs())); + // An expired cookie never hides a later live one. + (!is_expired(&expiry)).then_some((cookie.connect_id, expiry)) } impl IdentityProvider for ConnectIdProvider { @@ -194,8 +194,8 @@ mod tests { "connectId": "cid-example-1", "he": "example-hashed-email", "puid": "example-puid", - "lastSynced": 1_700_000_000_000_u64, - "lastUsed": 1_700_000_100_000_u64, + "lastSynced": 4_700_000_000_000_u64, + "lastUsed": 4_700_000_100_000_u64, "ttl": 86_400_000, }); @@ -208,28 +208,51 @@ mod tests { ); assert_eq!( expires_at(read), - 1_700_086_400, + 4_700_086_400, "should expire at lastSynced + ttl" ); } #[test] fn caps_the_ttl_and_defaults_a_missing_one() { - let long = json!({ "connectId": "cid", "lastSynced": 0, "ttl": 1e15 }); - let missing = json!({ "connectId": "cid", "lastSynced": 0 }); + // A far-future sync keeps the cookie live whatever today's date is. + let synced_secs = 4_700_000_000_u64; + let long = json!({ "connectId": "cid", "lastSynced": synced_secs * 1000, "ttl": 1e15 }); + let missing = json!({ "connectId": "cid", "lastSynced": synced_secs * 1000 }); assert_eq!( expires_at(read(&long)), - MAX_TTL.as_secs(), + synced_secs + MAX_TTL.as_secs(), "should cap at 14 days" ); assert_eq!( expires_at(read(&missing)), - DEFAULT_TTL.as_secs(), + synced_secs + DEFAULT_TTL.as_secs(), "should use the default" ); } + #[test] + fn an_expired_cookie_does_not_hide_a_later_live_one() { + let expired = json!({ "connectId": "cid-old", "lastSynced": 1000, "ttl": 1000 }); + let live = json!({ "connectId": "cid-live", "lastSynced": 4_700_000_000_000_u64 }); + let header = format!( + "connectId={}; connectId={}", + urlencoding::encode(&expired.to_string()), + urlencoding::encode(&live.to_string()) + ); + + let eids = provider() + .read_request_cookies(RequestCookies::new(&header)) + .into_found() + .expect("should read the live cookie"); + + assert_eq!( + eids[0].uids[0].id, "cid-live", + "should skip the expired cookie" + ); + } + #[test] fn ignores_missing_malformed_and_opt_out_cookies() { for cookie in [ diff --git a/crates/trusted-server-identity-providers/src/uid2.rs b/crates/trusted-server-identity-providers/src/uid2.rs index a8cf14ead..4f1352502 100644 --- a/crates/trusted-server-identity-providers/src/uid2.rs +++ b/crates/trusted-server-identity-providers/src/uid2.rs @@ -58,7 +58,7 @@ use url::Url; use crate::common::{ CookieReader, LockrSettings, ReadsLockr, bounded_expiry, cookie_capabilities, cookie_eid, - is_usable_id, percent_decoded, + is_expired, is_usable_id, percent_decoded, }; /// The UID2 provider kind. @@ -350,18 +350,20 @@ impl IdentityProvider for Uid2Provider { if opted_out { return CookieRead::OptedOut; } + // An expired identity never hides a later live one. let token = cookies - .get(&self.identity_cookie_name) - .and_then(identity) + .get_all(&self.identity_cookie_name) + .filter_map(identity) + .find(|token| !is_expired(&token.expiry)) .or_else(|| { self.cookies .read(cookies, |value| match token_cookie(value)? { - TokenCookie::Token(token, expiry) => Some(Token { + TokenCookie::Token(token, expiry) if !is_expired(&expiry) => Some(Token { token, expiry, refresh: None, }), - TokenCookie::OptedOut => None, + TokenCookie::Token(..) | TokenCookie::OptedOut => None, }) }); let Some(token) = token else { @@ -445,9 +447,9 @@ mod tests { let header = identity_cookie(&json!({ "advertising_token": "AgAAAAexampletoken+/=", "refresh_token": "example-refresh-token", - "identity_expires": 1_700_000_000_123_u64, - "refresh_from": 1_699_990_000_000_u64, - "refresh_expires": 1_702_000_000_000_u64, + "identity_expires": 4_700_000_000_123_u64, + "refresh_from": 4_699_990_000_000_u64, + "refresh_expires": 4_702_000_000_000_u64, "refresh_response_key": "example-response-key", })); @@ -458,7 +460,7 @@ mod tests { "should store only the token" ); assert!( - matches!(expiry, ProviderExpiry::At(at) if at.get() == 1_700_000_000), + matches!(expiry, ProviderExpiry::At(at) if at.get() == 4_700_000_000), "should expire at identity_expires" ); } @@ -494,9 +496,9 @@ mod tests { "advertising_token": "AgAAAAlatesttoken", "refresh_token": "example-refresh-token", "refresh_response_key": "example-response-key", - "identity_expires": 1_700_000_000_456_u64, - "refresh_from": 1_699_990_000_000_u64, - "refresh_expires": 1_702_000_000_000_u64, + "identity_expires": 4_700_000_000_456_u64, + "refresh_from": 4_699_990_000_000_u64, + "refresh_expires": 4_702_000_000_000_u64, }, })); @@ -507,7 +509,7 @@ mod tests { "should store only the latest advertising token" ); assert!( - matches!(expiry, ProviderExpiry::At(at) if at.get() == 1_700_000_000), + matches!(expiry, ProviderExpiry::At(at) if at.get() == 4_700_000_000), "should expire at the latest identity_expires" ); } @@ -517,7 +519,7 @@ mod tests { let provider = provider(&json!({})); let own = identity_cookie(&json!({ "advertising_token": "AgAAAAowntoken", - "identity_expires": 1_700_000_000_000_u64, + "identity_expires": 4_700_000_000_000_u64, })); let opt_out = prebid_cookie(&json!({ "originalToken": { "advertising_token": "AgAAAAoriginaltoken" }, @@ -534,6 +536,50 @@ mod tests { ); } + #[test] + fn reports_an_opt_out_in_a_later_duplicate_cookie() { + let provider = provider(&json!({})); + let token = prebid_cookie(&json!({ + "latestToken": { + "advertising_token": "AgAAAAlatesttoken", + "identity_expires": 4_700_000_000_000_u64, + }, + })); + let opt_out = prebid_cookie(&json!({ "latestToken": "optout" })); + let header = format!("{token}; {opt_out}"); + + let read = provider.read_request_cookies(RequestCookies::new(&header)); + + assert!( + matches!(read, CookieRead::OptedOut), + "should see the opt-out in a later cookie of the same name" + ); + } + + #[test] + fn an_expired_identity_does_not_hide_a_later_live_token() { + let provider = provider(&json!({})); + let expired = identity_cookie(&json!({ + "advertising_token": "AgAAAAexpiredtoken", + "identity_expires": 1_000_u64, + })); + let live = prebid_cookie(&json!({ + "latestToken": { + "advertising_token": "AgAAAAlivetoken", + "identity_expires": 4_700_000_000_000_u64, + }, + })); + let header = format!("{expired}; {live}"); + + let read = read(&*provider, &header); + + assert_eq!( + read.map(|(token, _)| token).as_deref(), + Some("AgAAAAlivetoken"), + "should skip the expired identity" + ); + } + #[test] fn never_stores_json_as_a_token() { let provider = provider(&json!({})); @@ -577,7 +623,7 @@ mod tests { })); let own = identity_cookie(&json!({ "advertising_token": "AgAAAAowntoken", - "identity_expires": 1_700_000_000_000_u64, + "identity_expires": 4_700_000_000_000_u64, })); let both = read(&*enabled, &format!("example_lockr_uid2=AgAAAAlockr; {own}")); @@ -632,9 +678,9 @@ mod tests { identity_cookie(&json!({ "advertising_token": "AgAAAAexampletoken", "refresh_token": "example-refresh-token", - "identity_expires": 1_700_000_000_000_u64, - "refresh_from": 1_699_990_000_000_u64, - "refresh_expires": 1_702_000_000_000_u64, + "identity_expires": 4_700_000_000_000_u64, + "refresh_from": 4_699_990_000_000_u64, + "refresh_expires": 4_702_000_000_000_u64, "refresh_response_key": BASE64.encode([7_u8; 32]), })) } @@ -662,7 +708,7 @@ mod tests { "should keep the refresh token" ); assert!( - matches!(from, ProviderExpiry::At(at) if at.get() == 1_699_990_000), + matches!(from, ProviderExpiry::At(at) if at.get() == 4_699_990_000), "should ask for a refresh at refresh_from" ); } @@ -686,15 +732,15 @@ mod tests { AgentType::try_from(3).expect("should build atype"), None, Some(ProviderId::new("uid2")), - Expiry::At(UnixSeconds::new(1_700_000_000)), + Expiry::At(UnixSeconds::new(4_700_000_000)), ) .with_refresh(KvRefresh { state, - from: UnixSeconds::new(1_699_990_000), + from: UnixSeconds::new(4_699_990_000), }); let record = KvPartnerId::from_eids( vec![KvEid::new(EidFields::default(), vec![uid])], - UnixSeconds::new(1_699_995_000), + UnixSeconds::new(4_699_995_000), ) .expect("should keep the live uid"); let mut entry = KvEntry::tombstone(1); @@ -797,9 +843,9 @@ mod tests { "body": { "advertising_token": "AgAAAArefreshedtoken", "refresh_token": "next-refresh-token", - "identity_expires": 1_700_100_000_000_u64, - "refresh_from": 1_700_090_000_000_u64, - "refresh_expires": 1_700_200_000_000_u64, + "identity_expires": 4_700_100_000_000_u64, + "refresh_from": 4_700_090_000_000_u64, + "refresh_expires": 4_700_200_000_000_u64, "refresh_response_key": BASE64.encode([9_u8; 32]), }, })); @@ -816,7 +862,7 @@ mod tests { let uid = &eids[0].uids[0]; assert_eq!(uid.id, "AgAAAArefreshedtoken", "should store the new token"); assert!( - matches!(uid.expiry, ProviderExpiry::At(at) if at.get() == 1_700_100_000), + matches!(uid.expiry, ProviderExpiry::At(at) if at.get() == 4_700_100_000), "should expire at the new identity_expires" ); let refresh = uid From 29c012de5c7e25d666cd04a878a74c234f639d42 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Fri, 6 Nov 2026 17:21:10 +0530 Subject: [PATCH 39/72] Refresh a due UID even while its cookie repeats it Also test merging cookie IDs with a failed, opted-out or found lookup. Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 185 ++++++++++++++++-- 1 file changed, 166 insertions(+), 19 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 1302cf1bc..d4d2e7008 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -272,6 +272,20 @@ impl ProviderCoordinator

{ self.providers.is_empty() } + /// Whether `source` holds a UID of this provider whose refresh is due. + fn refresh_due( + registered: &Registered

, + entry: Option<&KvEntry>, + source: &SourceDomain, + now: UnixSeconds, + ) -> bool { + entry + .and_then(|entry| entry.ids.get(source.as_str())) + .is_some_and(|record| { + record.is_provider_written() && record.refresh_due_for(®istered.id, now) + }) + } + /// Whether the row holds a live provider-written UID for `source`. fn holds_live_uid(entry: Option<&KvEntry>, source: &str, now: UnixSeconds) -> bool { entry @@ -576,8 +590,18 @@ impl IdentityCoordinator for ProviderCoordinator

{ // Only the sources the cookies filled are spared a // lookup; the provider's other due sources are // still looked up. + // A refresh-due source whose cookie only repeats the + // held UID stays due, so its refresh still runs before + // the UID expires. due.retain(|source| { - found.entries.iter().all(|(filled, _)| filled != source) + found + .entries + .iter() + .find(|(filled, _)| filled == source) + .is_none_or(|(_, eids)| { + Self::refresh_due(registered, entry, source, lookup.now) + && holds_same_uids_for(entry, source, eids, lookup.now) + }) }); // A cookie repeating the held UIDs is not rewritten, // so its relative expiry does not cause a write on @@ -807,24 +831,35 @@ fn holds_same_uids(entry: Option<&KvEntry>, found: &ProviderLookup, now: UnixSec let Some(entry) = entry else { return false; }; - found.entries.iter().all(|(source, eids)| { - let Some(record) = entry.ids.get(source.as_str()) else { - return false; - }; - let held: HashSet<&str> = record - .eids() - .iter() - .flat_map(KvEid::uids) - .filter(|uid| uid.is_live(now)) - .map(|uid| uid.id().as_str()) - .collect(); - let read: HashSet<&str> = eids - .iter() - .flat_map(KvEid::uids) - .map(|uid| uid.id().as_str()) - .collect(); - held == read - }) + found + .entries + .iter() + .all(|(source, eids)| holds_same_uids_for(Some(entry), source, eids, now)) +} + +/// Whether the row holds exactly the live UIDs in `eids` for `source`. +fn holds_same_uids_for( + entry: Option<&KvEntry>, + source: &SourceDomain, + eids: &[KvEid], + now: UnixSeconds, +) -> bool { + let Some(record) = entry.and_then(|entry| entry.ids.get(source.as_str())) else { + return false; + }; + let held: HashSet<&str> = record + .eids() + .iter() + .flat_map(KvEid::uids) + .filter(|uid| uid.is_live(now)) + .map(|uid| uid.id().as_str()) + .collect(); + let read: HashSet<&str> = eids + .iter() + .flat_map(KvEid::uids) + .map(|uid| uid.id().as_str()) + .collect(); + held == read } fn log_outcome(lookup: &ProviderLookup) { @@ -851,6 +886,7 @@ fn log_outcome(lookup: &ProviderLookup) { mod tests { use super::*; use crate::consent::types::TcfConsent; + use crate::ec::coordinator::results::DropCounts; use crate::ec::eid_types::{AgentType, EidFields, Expiry, UidValue}; use crate::ec::kv::KvIdentityGraph; use crate::ec::kv_types::{KvEid, KvPartnerId, KvRefresh, KvUid}; @@ -1873,6 +1909,117 @@ mod tests { assert_eq!(transport.sent.borrow().len(), 1, "should send the refresh"); } + #[test] + fn a_due_refresh_runs_while_the_cookie_repeats_the_held_uid() { + let coordinator = cookie_coordinator(); + let transport = CannedTransport::new(ok_reply); + + run_with_cookies( + &coordinator, + &transport, + &snapshot_holding(refreshable_record(900)), + &granting(), + &Limiter(false), + "vendor_id=p-1", + ); + + assert_eq!( + transport.sent.borrow().len(), + 1, + "should send the refresh although the cookie holds the same UID" + ); + } + + #[test] + fn a_newer_cookie_uid_replaces_a_refresh() { + let coordinator = cookie_coordinator(); + let transport = CannedTransport::new(ok_reply); + + run_with_cookies( + &coordinator, + &transport, + &snapshot_holding(refreshable_record(900)), + &granting(), + &Limiter(false), + "vendor_id=c-2", + ); + + assert!( + transport.sent.borrow().is_empty(), + "should take the newer cookie UID instead of refreshing" + ); + } + + fn found_lookup(source: &str, id: &str) -> ProviderLookup { + let uid = KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::Never, + ); + ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Found { + kept: 1, + dropped: DropCounts::default(), + }, + entries: vec![( + SourceDomain::try_from(source).expect("should build source"), + vec![KvEid::new(EidFields::default(), vec![uid])], + )], + cleared: Vec::new(), + } + } + + fn status_lookup(status: LookupStatus) -> ProviderLookup { + ProviderLookup { + provider: ProviderId::new("example-provider"), + status, + entries: Vec::new(), + cleared: Vec::new(), + } + } + + #[test] + fn merging_keeps_cookie_ids_when_the_lookup_fails_and_lets_an_opt_out_win() { + for failed in [ + LookupStatus::Timeout, + LookupStatus::HttpError(http::StatusCode::BAD_GATEWAY), + ] { + let merged = merge_lookups(found_lookup("a.example.com", "c-1"), status_lookup(failed)); + assert!( + matches!(merged.status, LookupStatus::Found { kept: 1, .. }), + "should keep the cookie IDs when the lookup fails" + ); + } + + let merged = merge_lookups( + found_lookup("a.example.com", "c-1"), + status_lookup(LookupStatus::Skipped(SkipReason::OptedOut)), + ); + + assert_eq!( + merged.status, + LookupStatus::Skipped(SkipReason::OptedOut), + "should let a lookup opt-out win over cookie IDs" + ); + } + + #[test] + fn merging_combines_cookie_and_lookup_sources() { + let merged = merge_lookups( + found_lookup("a.example.com", "c-1"), + found_lookup("b.example.com", "l-1"), + ); + + assert!( + matches!(merged.status, LookupStatus::Found { kept: 2, .. }), + "should count both sources" + ); + assert_eq!(merged.entries.len(), 2, "should keep both sources"); + } + #[test] fn a_future_refresh_leaves_the_source_held() { let coordinator = coordinator(""); From c26a97992b0a229337219ab9402db7deddbc6f12 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 02:35:51 +0530 Subject: [PATCH 40/72] Keep held cookie UIDs and refresh state while another source is due Drop cookie sources that repeat the held UIDs before queuing the cookie result, compare atype, ext and EID-level fields as well as IDs, and recount kept UIDs after filtering or exclusion. Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 183 ++++++++++++++---- 1 file changed, 146 insertions(+), 37 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index d4d2e7008..68ecf5800 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -25,9 +25,9 @@ use crate::consent::jurisdiction::Jurisdiction; use crate::consent::{allows_eid_persistence, tcf_vendor_consent_denied}; use crate::ec::EcKvSnapshot; use crate::ec::consent::ec_consent_granted; -use crate::ec::eid_types::{ProviderId, SourceDomain, UnixSeconds}; +use crate::ec::eid_types::{EidFields, ProviderId, SourceDomain, UnixSeconds}; use crate::ec::generation::ec_hash; -use crate::ec::kv_types::{KvEid, KvEntry}; +use crate::ec::kv_types::{KvEid, KvEntry, KvUid}; use crate::ec::provider::{ ClientSignals, CookieRead, IdentityProvider, LookupInput, LookupOutcome, OwnedEntries, ProviderConfigError, ProviderEid, ProviderResponse, RequestCookies, SourceInput, @@ -603,19 +603,24 @@ impl IdentityCoordinator for ProviderCoordinator

{ && holds_same_uids_for(entry, source, eids, lookup.now) }) }); - // A cookie repeating the held UIDs is not rewritten, - // so its relative expiry does not cause a write on - // every navigation. - if due.is_empty() && holds_same_uids(entry, &found, lookup.now) { - pending.early.push(skip(SkipReason::NotDue)); - continue; - } - pending.from_cookies.push(Tagged { - provider, - value: found, + // A source whose cookie repeats the held UIDs is not + // rewritten: its relative expiry would cause a write + // on every navigation, and a failed lookup for a + // still-due source would replace the held record and + // its refresh state. + let mut found = found; + found.entries.retain(|(source, eids)| { + !holds_same_uids_for(entry, source, eids, lookup.now) }); - if due.is_empty() { - continue; + recount_found(&mut found); + if !found.entries.is_empty() { + pending.from_cookies.push(Tagged { + provider, + value: found, + }); + if due.is_empty() { + continue; + } } } } @@ -776,9 +781,7 @@ fn exclude_sources(lookup: &mut ProviderLookup, sources: Vec) { lookup .entries .retain(|(source, _)| !sources.contains(source)); - if matches!(lookup.status, LookupStatus::Found { .. }) && lookup.entries.is_empty() { - lookup.status = LookupStatus::NoMatch; - } + recount_found(lookup); for source in sources { if !lookup.cleared.contains(&source) { lookup.cleared.push(source); @@ -825,19 +828,28 @@ fn merge_lookups(earlier: ProviderLookup, later: ProviderLookup) -> ProviderLook } } -/// Whether the row already holds exactly the UIDs `found` read from cookies, -/// for every source `found` returned. -fn holds_same_uids(entry: Option<&KvEntry>, found: &ProviderLookup, now: UnixSeconds) -> bool { - let Some(entry) = entry else { - return false; - }; - found - .entries - .iter() - .all(|(source, eids)| holds_same_uids_for(Some(entry), source, eids, now)) +/// Recounts the UIDs a found result keeps after some of its entries were +/// dropped; a result left with none becomes `NoMatch`. +fn recount_found(lookup: &mut ProviderLookup) { + if let LookupStatus::Found { kept, .. } = &mut lookup.status { + let remaining = lookup + .entries + .iter() + .flat_map(|(_, eids)| eids) + .map(|eid| eid.uids().len()) + .sum::(); + *kept = u16::try_from(remaining).unwrap_or(u16::MAX); + if remaining == 0 { + lookup.status = LookupStatus::NoMatch; + } + } } /// Whether the row holds exactly the live UIDs in `eids` for `source`. +/// +/// UIDs are compared by ID, agent type, `ext` and their EID-level fields; +/// only expiry and refresh state are ignored, so a cookie that changes any +/// stored field is still written. fn holds_same_uids_for( entry: Option<&KvEntry>, source: &SourceDomain, @@ -847,19 +859,30 @@ fn holds_same_uids_for( let Some(record) = entry.and_then(|entry| entry.ids.get(source.as_str())) else { return false; }; - let held: HashSet<&str> = record + let held: Vec<(&EidFields, &KvUid)> = record .eids() .iter() - .flat_map(KvEid::uids) - .filter(|uid| uid.is_live(now)) - .map(|uid| uid.id().as_str()) + .flat_map(|eid| eid.uids().iter().map(move |uid| (eid.fields(), uid))) + .filter(|(_, uid)| uid.is_live(now)) .collect(); - let read: HashSet<&str> = eids + let read: Vec<(&EidFields, &KvUid)> = eids .iter() - .flat_map(KvEid::uids) - .map(|uid| uid.id().as_str()) + .flat_map(|eid| eid.uids().iter().map(move |uid| (eid.fields(), uid))) .collect(); - held == read + let same = |(left_fields, left): &(&EidFields, &KvUid), + (right_fields, right): &(&EidFields, &KvUid)| { + left_fields == right_fields + && left.id() == right.id() + && left.atype() == right.atype() + && left.ext() == right.ext() + }; + held.len() == read.len() + && read + .iter() + .all(|read_uid| held.iter().any(|held_uid| same(held_uid, read_uid))) + && held + .iter() + .all(|held_uid| read.iter().any(|read_uid| same(held_uid, read_uid))) } fn log_outcome(lookup: &ProviderLookup) { @@ -1257,7 +1280,7 @@ mod tests { fn provider_record(id: &str) -> KvPartnerId { let uid = KvUid::new( UidValue::try_from(id).expect("should build uid"), - AgentType::try_from(1).expect("should build atype"), + AgentType::try_from(3).expect("should build atype"), None, Some(ProviderId::new("example-provider")), Expiry::Never, @@ -1867,7 +1890,7 @@ mod tests { fn refreshable_record_owned_by(owner: &str, from: u64) -> KvPartnerId { let uid = KvUid::new( UidValue::try_from("p-1").expect("should build uid"), - AgentType::try_from(1).expect("should build atype"), + AgentType::try_from(3).expect("should build atype"), None, Some(ProviderId::new(owner)), Expiry::At(UnixSeconds::new(5000)), @@ -2529,6 +2552,92 @@ mod tests { ); } + #[test] + fn an_unchanged_cookie_is_not_rewritten_while_another_source_is_due() { + let coordinator = two_source_coordinator(|provider| provider); + let replies: [Reply; 4] = [ + not_found_reply, + timeout_reply, + server_error_reply, + invalid_reply, + ]; + + for reply in replies { + let transport = CannedTransport::new(reply); + + let results = run(&coordinator, &transport, &held_snapshot("c-1"), &granting()); + + assert_eq!( + transport.sent.borrow().len(), + 1, + "should still look up the second source" + ); + let lookup = results.iter().next().expect("should report the provider"); + assert!( + lookup.entries.is_empty(), + "should not rewrite the cookie UID the row already holds: {:?}", + lookup.status + ); + } + } + + #[test] + fn a_failed_refresh_keeps_the_held_refresh_state() { + let coordinator = cookie_coordinator(); + let replies: [Reply; 3] = [timeout_reply, server_error_reply, invalid_reply]; + + for reply in replies { + let transport = CannedTransport::new(reply); + + let results = run_with_cookies( + &coordinator, + &transport, + &snapshot_holding(refreshable_record(900)), + &granting(), + &Limiter(false), + "vendor_id=p-1", + ); + + assert_eq!(transport.sent.borrow().len(), 1, "should send the refresh"); + let lookup = results.iter().next().expect("should report the provider"); + assert!( + lookup.entries.is_empty(), + "should not replace the held record after a failed refresh: {:?}", + lookup.status + ); + } + } + + #[test] + fn a_cookie_changing_only_the_agent_type_is_rewritten() { + let coordinator = cookie_coordinator(); + let transport = CannedTransport::new(ok_reply); + let uid = KvUid::new( + UidValue::try_from("c-1").expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::Never, + ); + let record = KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![uid])], + UnixSeconds::new(1000), + ) + .expect("should build provider record"); + + let results = run( + &coordinator, + &transport, + &snapshot_holding(record), + &granting(), + ); + + assert!( + matches!(only_status(&results), LookupStatus::Found { kept: 1, .. }), + "should store a cookie UID whose agent type changed" + ); + } + /// A two-source provider whose cookie source is excluded where GDPR /// applies. struct CookieSourceExcluded(TwoSourceProvider); From 8067db1b94a94574c4ad8bffbdbb76529cde7e5d Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 02:35:51 +0530 Subject: [PATCH 41/72] Hide cookie values from the RequestCookies Debug output Print only cookie names so a debug log never dumps session or consent cookies. Signed-off-by: dhruv8sh --- crates/trusted-server-core/src/ec/provider.rs | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs index 22ae55918..a4e9934ec 100644 --- a/crates/trusted-server-core/src/ec/provider.rs +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -166,9 +166,25 @@ impl<'cookie> From>>> for CookieRead<'cookie> { /// The request's `Cookie` header, lent to providers that read their own /// first-party cookies. -#[derive(Debug, Clone, Copy, Default)] +/// +/// Its `Debug` output names the cookies but never prints their values. +#[derive(Clone, Copy, Default)] pub struct RequestCookies<'a>(&'a str); +impl core::fmt::Debug for RequestCookies<'_> { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter + .debug_list() + .entries( + self.0 + .split(';') + .filter_map(|pair| pair.split_once('=')) + .map(|(name, _)| name.trim()), + ) + .finish() + } +} + impl<'a> RequestCookies<'a> { /// Wraps a `Cookie` header value (pairs separated by `;`). #[must_use] @@ -843,6 +859,18 @@ mod tests { ); } + #[test] + fn cookies_debug_prints_names_but_not_values() { + let cookies = RequestCookies::new("session=example-secret; vendor_id=v-1"); + + let printed = format!("{cookies:?}"); + + assert_eq!( + printed, r#"["session", "vendor_id"]"#, + "should print only cookie names" + ); + } + #[test] fn request_debug_hides_query_and_headers() { let mut headers = http::HeaderMap::new(); From 81f1e72ed4f56b2893dea966e879c0c3c3ae6f69 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 02:35:51 +0530 Subject: [PATCH 42/72] Store the percent-decoded Criteo bid ID that Prebid sends Decode cto_bidid and unwrap a criteoId object, as Prebid's criteoIdSystem does. Signed-off-by: dhruv8sh --- .../src/criteo.rs | 53 ++++++++++++++----- 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/crates/trusted-server-identity-providers/src/criteo.rs b/crates/trusted-server-identity-providers/src/criteo.rs index 8079f9e6e..0d77c0023 100644 --- a/crates/trusted-server-identity-providers/src/criteo.rs +++ b/crates/trusted-server-identity-providers/src/criteo.rs @@ -1,9 +1,11 @@ //! Criteo identity provider: reads the Criteo ID from the first-party //! cookie Prebid's `criteoIdSystem` writes. //! -//! Prebid stores Criteo's bid ID in `cto_bidid` and sends that value -//! unchanged (still percent-encoded) as the `criteo.com` EID with agent -//! type 1, so it is stored verbatim. No server call is made. +//! Prebid stores Criteo's bid ID in `cto_bidid` with `encodeURIComponent` +//! and sends the decoded value as the `criteo.com` EID with agent type 1, +//! so the cookie is percent-decoded before it is stored. Like Prebid's +//! `normalizeBidId`, a `{"criteoId":…}` value is unwrapped. No server call +//! is made. use std::borrow::Cow; @@ -16,7 +18,9 @@ use trusted_server_core::ec::provider::{ RequestCookies, }; -use crate::common::{CookieReader, bounded_expiry, cookie_capabilities, device_eid, is_usable_id}; +use crate::common::{ + CookieReader, bounded_expiry, cookie_capabilities, device_eid, is_usable_id, percent_decoded, +}; /// The Criteo provider kind. pub struct Criteo; @@ -89,20 +93,30 @@ impl IdentityProvider for CriteoProvider { &self, cookies: RequestCookies<'cookie>, ) -> CookieRead<'cookie> { - let Some(id) = self - .cookies - .read(cookies, |value| is_usable_id(value).then_some(value.trim())) - else { + let Some(id) = self.cookies.read(cookies, |value| { + let id = bid_id(&percent_decoded(value))?; + is_usable_id(&id).then(|| id.trim().to_owned()) + }) else { return CookieRead::Absent; }; CookieRead::Found(vec![device_eid( &self.source, - Cow::Borrowed(id), + Cow::Owned(id), bounded_expiry(None), )]) } } +/// The bid ID in a decoded `cto_bidid` value, unwrapping the +/// `{"criteoId":…}` form Prebid's `normalizeBidId` accepts. +fn bid_id(decoded: &str) -> Option { + if decoded.trim_start().starts_with('{') { + let value: serde_json::Value = serde_json::from_str(decoded).ok()?; + return value["criteoId"].as_str().map(str::to_owned); + } + Some(decoded.to_owned()) +} + #[cfg(test)] mod tests { use serde_json::json; @@ -122,7 +136,7 @@ mod tests { } #[test] - fn stores_the_bid_id_verbatim() { + fn stores_the_decoded_bid_id() { let eids = provider() .read_request_cookies(RequestCookies::new("cto_bidid=bidid-example%3D%3D")) .into_found() @@ -134,12 +148,27 @@ mod tests { ); let uid = &eids[0].uids[0]; assert_eq!( - uid.id, "bidid-example%3D%3D", - "should keep the value encoded" + uid.id, "bidid-example==", + "should store the decoded value Prebid sends" ); assert_eq!(uid.atype, Some(1), "should be device-based"); } + #[test] + fn unwraps_a_criteo_id_object() { + let eids = provider() + .read_request_cookies(RequestCookies::new( + "cto_bidid=%7B%22criteoId%22%3A%22bidid-example%22%7D", + )) + .into_found() + .expect("should read the bid ID"); + + assert_eq!( + eids[0].uids[0].id, "bidid-example", + "should store the wrapped bid ID" + ); + } + #[test] fn ignores_missing_and_blank_cookies() { for header in ["cto_bidid=", "cto_bidid=0", "cto_bundle=bundle-example"] { From 35a07e9ba38c66df1b15a64ef43b0e036baba2b7 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 02:35:51 +0530 Subject: [PATCH 43/72] Parse LiveRamp v2 envelope replies and send server-to-server atype Take the type 19 RampID entry from the envelopes array, send atype 3, and send only the first GPP section as gpp_sid. Signed-off-by: dhruv8sh --- .../src/liveramp.rs | 77 ++++++++++++++----- 1 file changed, 59 insertions(+), 18 deletions(-) diff --git a/crates/trusted-server-identity-providers/src/liveramp.rs b/crates/trusted-server-identity-providers/src/liveramp.rs index fd30bb820..ab1f66f57 100644 --- a/crates/trusted-server-identity-providers/src/liveramp.rs +++ b/crates/trusted-server-identity-providers/src/liveramp.rs @@ -13,16 +13,19 @@ //! `/api/identity/v2/envelope`), a visitor without an envelope cookie is looked up server side by Custom ID (`it=15`): the //! stable part of the EC ID, sent as `iv=:` with //! the placement ID, consent strings, the publisher `Origin` and the -//! visitor's IP in `X-Forwarded-For`. `LiveRamp` must enable Custom ID for -//! the placement. A `204` reply reports an opt-out. Without an `endpoint` -//! no server call is made. +//! visitor's IP in `X-Forwarded-For`, as a server-to-server call +//! (`atype=3`). `LiveRamp` must enable Custom ID for the placement. A `200` +//! reply lists `envelopes`; the `RampID` entry (`type` 19) is stored. A +//! `204` reply reports an opt-out. Without an `endpoint` no server call is +//! made. //! //! With the Lockr capability enabled (`[settings.lockr]`), the cookies Lockr //! AIM writes the envelope into (default key `_lr_env`) are read after the //! own cookies, with the same parser. //! //! Parsing is lenient: a value outside the base64 alphabet, a JSON body -//! without a string `envelope`, or a blank or opt-out envelope reads as no ID. +//! without a string `envelope`, or a blank or placeholder (`0`) envelope reads +//! as no ID. use std::borrow::Cow; use std::time::Duration; @@ -45,13 +48,15 @@ use trusted_server_core::ec::provider::{ use crate::common::{ CookieReader, LockrSettings, ReadsLockr, bounded_expiry, cookie_capabilities, cookie_eid, - is_routable_ip, is_usable_id, join_section_ids, percent_decoded, + is_routable_ip, is_usable_id, percent_decoded, }; /// `it` value of a Custom ID lookup. const CUSTOM_ID_TYPE: &str = "15"; -/// `atype` value of an ATS API integration. -const ATS_API: &str = "2"; +/// `atype` value of a server-to-server integration (`2` is the ATS API). +const SERVER_TO_SERVER: &str = "3"; +/// `type` of the `RampID` envelope in a v2 reply's `envelopes`. +const RAMPID_ENVELOPE_TYPE: u32 = 19; /// `ct` value of a TCF v2 consent string. const TCF_CONSENT: &str = "4"; /// `ct` value of a US Privacy consent string. @@ -160,6 +165,22 @@ struct EnvelopeCookie { envelope: String, } +/// A v2 envelope API reply: `{"envelopes":[{"type":19,"value":…}]}`. +#[derive(Deserialize)] +struct EnvelopeReply { + #[serde(default)] + envelopes: Vec, +} + +/// One entry of [`EnvelopeReply::envelopes`]. +#[derive(Deserialize)] +struct ReplyEnvelope { + #[serde(rename = "type")] + kind: u32, + #[serde(default)] + value: Option, +} + /// Decodes one cookie value into its envelope. /// /// A value whose base64 decodes to JSON must carry a usable `envelope` @@ -222,7 +243,7 @@ impl IdentityProvider for LiveRampProvider { "iv", &format!("{}:{stable_id}", envelope_lookup.custom_id_namespace), ) - .append_pair("atype", ATS_API); + .append_pair("atype", SERVER_TO_SERVER); if let Some(tc_string) = &consent.raw_tc_string { query .append_pair("ct", TCF_CONSENT) @@ -235,8 +256,10 @@ impl IdentityProvider for LiveRampProvider { if let Some(gpp) = &consent.raw_gpp_string { query.append_pair("gpp", gpp); } - if let Some(section_ids) = &consent.gpp_section_ids { - query.append_pair("gpp_sid", &join_section_ids(section_ids)); + // `gpp_sid` is a single section ID; Prebid also sends only the + // first applicable section. + if let Some(section_id) = consent.gpp_section_ids.as_deref().and_then(<[u16]>::first) { + query.append_pair("gpp_sid", §ion_id.to_string()); } } let mut headers = HeaderMap::new(); @@ -269,14 +292,20 @@ impl IdentityProvider for LiveRampProvider { return Err(Report::new(ProviderError::InvalidResponse) .attach(format!("liveramp returned status {}", response.status))); } - let reply: EnvelopeCookie = + let reply: EnvelopeReply = serde_json::from_slice(response.body).change_context(ProviderError::InvalidResponse)?; - if !is_usable_id(&reply.envelope) { + let Some(envelope) = reply + .envelopes + .into_iter() + .find(|envelope| envelope.kind == RAMPID_ENVELOPE_TYPE) + .and_then(|envelope| envelope.value) + .filter(|envelope| is_usable_id(envelope)) + else { return Ok(LookupOutcome::NoMatch); - } + }; Ok(LookupOutcome::Found(vec![cookie_eid( &self.source, - Cow::Owned(reply.envelope), + Cow::Owned(envelope), bounded_expiry(Some(ENVELOPE_TTL)), )])) } @@ -617,11 +646,11 @@ mod tests { ("pid", "14"), ("it", "15"), ("iv", "ana-example:abc123"), - ("atype", "2"), + ("atype", "3"), ("ct", "4"), ("cv", "CP_example"), ("gpp", "DBA_example"), - ("gpp_sid", "2,7"), + ("gpp_sid", "2"), ]), "should send the stable EC ID as the Custom ID with consent" ); @@ -685,7 +714,11 @@ mod tests { #[test] fn stores_the_returned_envelope() { - let body = json!({"envelope": "AhIdEnvelope"}).to_string(); + let body = json!({"envelopes": [ + {"type": 25, "source": "pairIds", "value": "pair-example", "err": null}, + {"type": 19, "source": "envelopeLiveramp", "value": "AhIdEnvelope", "err": null}, + ]}) + .to_string(); let outcome = reply(StatusCode::OK, body.as_bytes()).expect("should parse reply"); @@ -709,8 +742,16 @@ mod tests { #[test] fn maps_no_content_to_an_opt_out_and_a_blank_envelope_to_no_match() { - let blank = json!({"envelope": ""}).to_string(); + let blank = json!({"envelopes": [{"type": 19, "value": "", "err": null}]}).to_string(); + let other = json!({"envelopes": [{"type": 25, "value": "pair-example"}]}).to_string(); + assert!( + matches!( + reply(StatusCode::OK, other.as_bytes()), + Ok(LookupOutcome::NoMatch) + ), + "should report no match without a RampID envelope" + ); assert!( matches!( reply(StatusCode::NO_CONTENT, b""), From 1c2973f5de74eda0df620cb1dbfb6a7d77f1afe8 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 02:35:51 +0530 Subject: [PATCH 44/72] Send ID5 gpp_sid comma-joined and IPv6 addresses as ipv6 Match ID5's server-side client, detect the opt-out on the reply as sent without percent-decoding it, and drop the duplicate opt-out constant. Signed-off-by: dhruv8sh --- .../src/id5.rs | 97 ++++++++++++++----- 1 file changed, 74 insertions(+), 23 deletions(-) diff --git a/crates/trusted-server-identity-providers/src/id5.rs b/crates/trusted-server-identity-providers/src/id5.rs index 00bfa6eca..519b70427 100644 --- a/crates/trusted-server-identity-providers/src/id5.rs +++ b/crates/trusted-server-identity-providers/src/id5.rs @@ -27,12 +27,16 @@ //! removed. A fetch reply whose `universal_uid` is `0` reports the same //! opt-out. //! -//! The request field names for the client IP and user agent follow ID5's -//! server-to-server integration and should be confirmed with ID5 for each -//! partner account. +//! `gpp_sid` is sent comma-joined, as ID5's own server-side client does. +//! The client IP goes in `ip` (IPv4) or `ipv6`. The request field names for +//! the client IP and user agent, and the `o`, `v`, `tml` and `nbPage` +//! fields (ID5's server-side client sends `provider`, `domain` and `origin` +//! instead), follow ID5's server-to-server integration and should be +//! confirmed with ID5 for each partner account. use std::borrow::Cow; use std::collections::BTreeMap; +use std::net::IpAddr; use std::num::NonZeroUsize; use std::time::Duration; @@ -56,7 +60,7 @@ use serde_json::value::RawValue; use crate::common::{ CookieReader, LockrSettings, ReadsLockr, VendorEid, bounded_expiry, is_routable_ip, - is_usable_id, percent_decoded, + is_usable_id, join_section_ids, percent_decoded, }; /// Formats the current time as the RFC 3339 `ts` the `/gs/v2` fetch requires. @@ -180,13 +184,15 @@ struct FetchBody<'a> { #[serde(skip_serializing_if = "Option::is_none")] gpp_string: Option<&'a str>, #[serde(skip_serializing_if = "Option::is_none")] - gpp_sid: Option<&'a [u16]>, + gpp_sid: Option, tml: &'a str, ts: String, #[serde(skip_serializing_if = "Option::is_none")] ua: Option<&'a str>, #[serde(skip_serializing_if = "Option::is_none")] ip: Option, + #[serde(skip_serializing_if = "Option::is_none")] + ipv6: Option, } /// The top-level reply; each field is parsed separately so one bad shape @@ -344,6 +350,7 @@ impl IdentityProvider for Id5Provider { ) -> Result> { let consent = lookup.consent; let origin = format!("https://{}", lookup.publisher_domain); + let routable_ip = lookup.client.ip.filter(|ip| is_routable_ip(*ip)); let body = FetchBody { partner: self.partner_id, v: env!("CARGO_PKG_VERSION"), @@ -353,15 +360,12 @@ impl IdentityProvider for Id5Provider { gdpr_consent: consent.raw_tc_string.as_deref(), us_privacy: consent.raw_us_privacy.as_deref(), gpp_string: consent.raw_gpp_string.as_deref(), - gpp_sid: consent.gpp_section_ids.as_deref(), + gpp_sid: consent.gpp_section_ids.as_deref().map(join_section_ids), tml: &origin, ts: request_timestamp()?, ua: lookup.client.user_agent, - ip: lookup - .client - .ip - .filter(|ip| is_routable_ip(*ip)) - .map(|ip| ip.to_string()), + ip: routable_ip.filter(IpAddr::is_ipv4).map(|ip| ip.to_string()), + ipv6: routable_ip.filter(IpAddr::is_ipv6).map(|ip| ip.to_string()), }; let body = serde_json::to_vec(&body).change_context(ProviderError::Request)?; let mut headers = HeaderMap::new(); @@ -387,7 +391,7 @@ impl IdentityProvider for Id5Provider { return Err(Report::new(ProviderError::InvalidResponse) .attach(format!("ID5 returned status {}", response.status))); } - if std::str::from_utf8(response.body).is_ok_and(is_opt_out_cookie) { + if is_opt_out_reply(response.body) { return Ok(LookupOutcome::OptedOut); } let eids = self @@ -411,24 +415,26 @@ impl IdentityProvider for Id5Provider { } } -/// The ID5 opt-out UID. -const OPT_OUT_UID: &str = "0"; - -/// Whether one ID5 cookie value or fetch reply records an opt-out: a bare -/// `0`, or a reply whose `universal_uid` is `0`. +/// Whether one ID5 cookie value records an opt-out: a bare `0`, or a +/// (possibly percent-encoded) reply whose `universal_uid` is `0`. fn is_opt_out_cookie(value: &str) -> bool { let decoded = percent_decoded(value); let decoded = decoded.trim(); if !decoded.starts_with('{') { - return decoded == OPT_OUT_UID; + return decoded == OPTED_OUT_UID; } - serde_json::from_str::(decoded) + is_opt_out_reply(decoded.as_bytes()) +} + +/// Whether a fetch reply body, taken as is, has a `universal_uid` of `0`. +fn is_opt_out_reply(body: &[u8]) -> bool { + serde_json::from_slice::(body) .ok() .and_then(|reply| { reply .get("universal_uid") .and_then(serde_json::Value::as_str) - .map(|uid| uid == OPT_OUT_UID) + .map(|uid| uid == OPTED_OUT_UID) }) .unwrap_or(false) } @@ -545,9 +551,8 @@ mod tests { assert_eq!(body["gdpr"], 0, "should send that GDPR does not apply"); assert_eq!(body["us_privacy"], "1YNN", "should send US privacy"); assert_eq!( - body["gpp_sid"], - json!([7, 8]), - "should send GPP section IDs as an array" + body["gpp_sid"], "7,8", + "should send GPP section IDs comma-joined" ); assert_eq!(body["ip"], "11.22.33.44", "should send the client IP"); assert_eq!( @@ -583,6 +588,34 @@ mod tests { assert!(body.get("ip").is_none(), "should not send a private IP"); } + #[test] + fn sends_an_ipv6_address_as_ipv6() { + let provider = provider(); + let consent = ConsentContext::default(); + let lookup = LookupInput { + ec_id: "ec-example", + consent: &consent, + client: ClientSignals { + ip: Some("2001:4860::1".parse().expect("should parse ip")), + user_agent: None, + }, + publisher_domain: "www.example.com", + due_sources: &[], + current: OwnedEntries::empty(), + cookies: RequestCookies::default(), + hashed_email: None, + }; + + let request = provider + .build_request(&lookup) + .expect("should build request"); + let body: serde_json::Value = + serde_json::from_slice(&request.body).expect("should send JSON"); + + assert!(body.get("ip").is_none(), "should not send IPv6 as ip"); + assert_eq!(body["ipv6"], "2001:4860::1", "should send ipv6"); + } + #[test] fn parses_complete_eids_for_configured_sources() { let provider = provider(); @@ -707,6 +740,24 @@ mod tests { ); } + #[test] + fn does_not_percent_decode_an_opt_out_reply() { + let provider = provider(); + let headers = HeaderMap::new(); + let body = serde_json::to_vec(&json!({ + "universal_uid": "0", + "ext": { "note": "a%22b" }, + })) + .expect("should encode body"); + + let outcome = respond(&*provider, &headers, &body); + + assert!( + matches!(outcome, LookupOutcome::OptedOut), + "should read the reply as sent" + ); + } + #[test] fn rejects_an_error_status_and_a_malformed_body() { let provider = provider(); From cdca7bb0276584c062787da7af1c5688948a62cd Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 02:35:51 +0530 Subject: [PATCH 45/72] Honour the UID2 SDK opt-out identity cookie A status of optout in any __uid_2 duplicate now reports an opt-out over every token. The module doc now says GDPR regions skip UID2 for region. Signed-off-by: dhruv8sh --- .../src/uid2.rs | 42 ++++++++++++++++--- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/crates/trusted-server-identity-providers/src/uid2.rs b/crates/trusted-server-identity-providers/src/uid2.rs index 4f1352502..aa25ca758 100644 --- a/crates/trusted-server-identity-providers/src/uid2.rs +++ b/crates/trusted-server-identity-providers/src/uid2.rs @@ -28,14 +28,16 @@ //! (default key `__uid2_advertising_token`) are read last, with the same //! parser. //! -//! A `latestToken` of `"optout"` in any of those cookies reports an opt-out: -//! no lookup is made and stored UID2 UIDs are removed. +//! A `{"status":"optout"}` identity in `__uid_2` (the UID2 SDK's opt-out) +//! or a `latestToken` of `"optout"` in any of those cookies reports an +//! opt-out: no lookup is made and stored UID2 UIDs are removed. //! //! UID2 is not offered where GDPR applies: there the provider is skipped -//! for consent, so its cookies are not read and no refresh is made. +//! for region, so its cookies are not read, no refresh is made and the UIDs +//! the row holds for it are removed. //! //! Parsing is lenient: an undecodable cookie, a missing token or expiry, -//! or a blank or opt-out token reads as no ID. A JSON value is never stored +//! or a blank or placeholder (`0`) token reads as no ID. A JSON value is never stored //! as a token. use std::borrow::Cow; @@ -344,7 +346,13 @@ impl IdentityProvider for Uid2Provider { &self, cookies: RequestCookies<'cookie>, ) -> CookieRead<'cookie> { - let opted_out = self.cookies.any(cookies, |value| { + // The UID2 SDK records an opt-out as `{"status":"optout"}` in its + // identity cookie, Prebid as a `latestToken` of `"optout"`; either + // one, in any duplicate cookie, wins over every token. + let opted_out = cookies.get_all(&self.identity_cookie_name).any(|value| { + serde_json::from_str::(&percent_decoded(value)) + .is_ok_and(|identity| identity["status"] == REFRESH_OPT_OUT) + }) || self.cookies.any(cookies, |value| { matches!(token_cookie(value), Some(TokenCookie::OptedOut)) }); if opted_out { @@ -536,6 +544,30 @@ mod tests { ); } + #[test] + fn reports_an_sdk_opt_out_over_a_fallback_token() { + let provider = provider(&json!({})); + let live = identity_cookie(&json!({ + "advertising_token": "AgAAAAowntoken", + "identity_expires": 4_700_000_000_000_u64, + })); + let opt_out = identity_cookie(&json!({ "status": "optout" })); + let fallback = prebid_cookie(&json!({ + "latestToken": { + "advertising_token": "AgAAAAlatesttoken", + "identity_expires": 4_700_000_000_000_u64, + }, + })); + let header = format!("{live}; {opt_out}; {fallback}"); + + let read = provider.read_request_cookies(RequestCookies::new(&header)); + + assert!( + matches!(read, CookieRead::OptedOut), + "should report the SDK opt-out over every token" + ); + } + #[test] fn reports_an_opt_out_in_a_later_duplicate_cookie() { let provider = provider(&json!({})); From 69c62e125a02b19a4e1fcf5e3d27dce807174e46 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 02:35:51 +0530 Subject: [PATCH 46/72] Send LiveIntent's TDID as atype 1 and keep nonId when segments are too large Match Prebid's agent type for the resolved Trade Desk ID, and leave out a segments ext over MAX_EXT_BYTES instead of losing the whole nonId EID. Signed-off-by: dhruv8sh --- .../src/liveintent.rs | 64 +++++++++++++++++-- 1 file changed, 58 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-identity-providers/src/liveintent.rs b/crates/trusted-server-identity-providers/src/liveintent.rs index c37e6ef10..cade56c94 100644 --- a/crates/trusted-server-identity-providers/src/liveintent.rs +++ b/crates/trusted-server-identity-providers/src/liveintent.rs @@ -29,7 +29,7 @@ use http::header::{DATE, EXPIRES, ORIGIN}; use http::{HeaderMap, HeaderValue, Method, StatusCode}; use serde::Deserialize; use serde_json::value::RawValue; -use trusted_server_core::ec::eid_types::SourceDomain; +use trusted_server_core::ec::eid_types::{MAX_EXT_BYTES, SourceDomain}; use trusted_server_core::ec::provider::{ EidLevelFields, EidShape, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, ProviderConfigError, ProviderEid, ProviderError, ProviderExpiry, ProviderKind, @@ -37,7 +37,7 @@ use trusted_server_core::ec::provider::{ }; use url::Url; -use crate::common::{bounded_expiry, is_routable_ip, join_section_ids}; +use crate::common::{DEVICE_ATYPE, bounded_expiry, is_routable_ip, join_section_ids}; /// Each attribute `LiveIntent` resolves and the source it carries. pub const ATTRIBUTE_SOURCES: &[(&str, &str)] = &[ @@ -343,6 +343,17 @@ impl IdentityProvider for LiveIntentProvider { }) .and_then(|segments| { RawValue::from_string(format!(r#"{{"segments":{}}}"#, segments.get())).ok() + }) + // An oversized EID `ext` would drop the whole `nonId` EID, so the + // segments are left out rather than the ID. + .filter(|ext| { + let fits = ext.get().len() <= MAX_EXT_BYTES; + if !fits { + log::debug!( + "LiveIntent: omitting segments over {MAX_EXT_BYTES} bytes from the nonId EID" + ); + } + fits }); let eids: Vec> = self .attributes @@ -352,14 +363,22 @@ impl IdentityProvider for LiveIntentProvider { let id = Cow::<'body, str>::deserialize(*raw) .ok() .filter(|id| !id.is_empty())?; - let (matcher, eid_ext, uid_ext) = match attribute.as_str() { - NON_ID_ATTRIBUTE => (None, segments.clone().map(Cow::Owned), None), + // Prebid sends the Trade Desk ID LiveIntent resolved as a + // device-based ID (atype 1), like Unified ID's TDID. + let (matcher, eid_ext, uid_ext, atype) = match attribute.as_str() { + NON_ID_ATTRIBUTE => ( + None, + segments.clone().map(Cow::Owned), + None, + LIVEINTENT_ATYPE, + ), TDID_ATTRIBUTE => ( Some(Cow::Borrowed(TDID_MATCHER)), None, Some(self.tdid_uid_ext), + DEVICE_ATYPE, ), - _ => (None, None, Some(self.partner_uid_ext)), + _ => (None, None, Some(self.partner_uid_ext), LIVEINTENT_ATYPE), }; Some(ProviderEid { source: Cow::Owned(source.as_str().to_owned()), @@ -369,7 +388,7 @@ impl IdentityProvider for LiveIntentProvider { ext: eid_ext, uids: vec![ProviderUid { id, - atype: Some(LIVEINTENT_ATYPE), + atype: Some(atype), ext: uid_ext.map(Cow::Borrowed), expiry, refresh: None, @@ -584,6 +603,34 @@ mod tests { } } + #[test] + fn omits_oversized_segments_but_keeps_the_non_id() { + let provider = provider(); + let headers = HeaderMap::new(); + let segments: Vec = (0..120).map(|n| format!("seg{n:04}")).collect(); + let body = json!({ "nonId": "li-example", "segments": segments }) + .to_string() + .into_bytes(); + + let outcome = provider + .parse_response(ProviderResponse { + status: StatusCode::OK, + headers: &headers, + body: &body, + context: None, + }) + .expect("should parse response"); + let LookupOutcome::Found(eids) = outcome else { + panic!("should find the nonId EID"); + }; + + assert_eq!(eids[0].uids[0].id, "li-example", "should keep the ID"); + assert!( + eids[0].ext.is_none(), + "should omit segments that exceed the ext limit" + ); + } + #[test] fn maps_returned_attributes_to_sources_with_the_expires_header() { let provider = provider(); @@ -643,6 +690,11 @@ mod tests { "should keep nonId without matcher" ); assert_eq!(eids[1].source, "adserver.org", "should map the TDID"); + assert_eq!( + eids[1].uids[0].atype, + Some(1), + "should send the TDID as device-based, as Prebid does" + ); assert_eq!( eids[1].matcher.as_deref(), Some("liveintent.com"), From 996ca33e4ed4b89dac8edb154c095aab4dda6965 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 02:35:51 +0530 Subject: [PATCH 47/72] Match Prebid's ConnectID cookie validity rules Treat a missing or zero ttl as 14 days, read a missing lastSynced as fresh, and accept the lowercase connectid key. Signed-off-by: dhruv8sh --- .../src/connectid.rs | 56 ++++++++++++++----- 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/crates/trusted-server-identity-providers/src/connectid.rs b/crates/trusted-server-identity-providers/src/connectid.rs index da12f8244..f7441efa3 100644 --- a/crates/trusted-server-identity-providers/src/connectid.rs +++ b/crates/trusted-server-identity-providers/src/connectid.rs @@ -5,8 +5,10 @@ //! (optionally percent-encoded) with `connectId`, `lastSynced` and `ttl`, //! both in milliseconds. The ID is stored with agent type 3 until //! `lastSynced + ttl`, with the TTL capped at [`MAX_TTL`]; the coordinator -//! drops it once that time has passed. A missing `ttl` uses the shared -//! default TTL. No server call is made. +//! drops it once that time has passed. As in Prebid's `connectIdSystem`, a +//! missing or zero `ttl` means [`MAX_TTL`], a missing `lastSynced` means +//! the ID is fresh (stored for [`MAX_TTL`]), and the key may also be spelled +//! `connectid`. No server call is made. //! //! Yahoo's opt-out (`connectIdOptOut` = `1`) is kept in local storage, which //! a server never sees, so it is not honoured here: a `ConnectID` cookie left @@ -17,7 +19,7 @@ //! //! The cookie's `he` and `puid` fields are never read, so they are never //! stored. Parsing is lenient: an undecodable cookie, a missing -//! `connectId` or `lastSynced`, or a blank or opt-out ID reads as no ID. +//! `connectId`, or a blank or placeholder (`0`) ID reads as no ID. use std::borrow::Cow; use std::time::Duration; @@ -32,8 +34,7 @@ use trusted_server_core::ec::provider::{ }; use crate::common::{ - CookieReader, DEFAULT_TTL, cookie_capabilities, cookie_eid, is_expired, is_usable_id, - percent_decoded, + CookieReader, cookie_capabilities, cookie_eid, is_expired, is_usable_id, percent_decoded, }; /// Longest time a `ConnectID` is kept after its last sync. @@ -91,10 +92,10 @@ impl ProviderKind for ConnectId { /// The fields of the `connectId` cookie this provider reads. #[derive(Deserialize)] struct ConnectIdCookie { - #[serde(rename = "connectId")] + #[serde(rename = "connectId", alias = "connectid")] connect_id: String, - #[serde(rename = "lastSynced")] - last_synced_ms: f64, + #[serde(rename = "lastSynced", default)] + last_synced_ms: Option, #[serde(default)] ttl: Option, } @@ -111,8 +112,15 @@ fn connect_id(value: &str) -> Option<(String, ProviderExpiry)> { if !is_usable_id(&cookie.connect_id) { return None; } - let ttl = cookie.ttl.map_or(DEFAULT_TTL, from_millis).min(MAX_TTL); - let expires = from_millis(cookie.last_synced_ms).saturating_add(ttl); + let Some(last_synced_ms) = cookie.last_synced_ms else { + return Some((cookie.connect_id, ProviderExpiry::After(MAX_TTL))); + }; + let ttl = cookie + .ttl + .filter(|ttl| *ttl > 0.0) + .map_or(MAX_TTL, from_millis) + .min(MAX_TTL); + let expires = from_millis(last_synced_ms).saturating_add(ttl); let expiry = ProviderExpiry::At(UnixSeconds::new(expires.as_secs())); // An expired cookie never hides a later live one. (!is_expired(&expiry)).then_some((cookie.connect_id, expiry)) @@ -214,21 +222,40 @@ mod tests { } #[test] - fn caps_the_ttl_and_defaults_a_missing_one() { + fn caps_the_ttl_and_defaults_a_missing_or_zero_one() { // A far-future sync keeps the cookie live whatever today's date is. let synced_secs = 4_700_000_000_u64; let long = json!({ "connectId": "cid", "lastSynced": synced_secs * 1000, "ttl": 1e15 }); let missing = json!({ "connectId": "cid", "lastSynced": synced_secs * 1000 }); + let zero = json!({ "connectId": "cid", "lastSynced": synced_secs * 1000, "ttl": 0 }); assert_eq!( expires_at(read(&long)), synced_secs + MAX_TTL.as_secs(), "should cap at 14 days" ); + for cookie in [missing, zero] { + assert_eq!( + expires_at(read(&cookie)), + synced_secs + MAX_TTL.as_secs(), + "should use Prebid's 14 days for {cookie}" + ); + } + } + + #[test] + fn reads_a_lowercase_key_and_a_missing_last_sync_as_fresh() { + let lowercase = json!({ "connectid": "cid", "lastSynced": 4_700_000_000_000_u64 }); + let unsynced = json!({ "connectId": "cid" }); + assert_eq!( - expires_at(read(&missing)), - synced_secs + DEFAULT_TTL.as_secs(), - "should use the default" + read(&lowercase).map(|(id, _)| id).as_deref(), + Some("cid"), + "should accept the lowercase key" + ); + assert!( + matches!(read(&unsynced), Some((_, ProviderExpiry::After(ttl))) if ttl == MAX_TTL), + "should store an unsynced ID for 14 days" ); } @@ -257,7 +284,6 @@ mod tests { fn ignores_missing_malformed_and_opt_out_cookies() { for cookie in [ json!({}), - json!({ "connectId": "cid" }), json!({ "connectId": "", "lastSynced": 1 }), json!({ "connectId": "0", "lastSynced": 1 }), json!({ "connectId": 5, "lastSynced": 1 }), From 7b60dd63e9c2507ef5130221df6e3e9aace6dbd9 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 02:35:51 +0530 Subject: [PATCH 48/72] Align provider docs, plans and example config with the code Correct the UID2, LiveRamp, ID5, ConnectID, LiveIntent and Criteo plans. Describe placeholder IDs as no ID rather than an opt-out, name the client settings that put tokens in cookies, and mark allowed_domains as required only with an endpoint. Signed-off-by: dhruv8sh --- .../trusted-server-identity-providers/src/common.rs | 8 ++++++-- docs/guide/configuration.md | 2 +- .../plans/2026-11-02-identity-provider-connectid.md | 4 ++-- .../plans/2026-11-02-identity-provider-criteo.md | 6 +++--- .../plans/2026-11-02-identity-provider-id5.md | 4 ++-- .../2026-11-02-identity-provider-liveintent.md | 4 ++-- .../plans/2026-11-02-identity-provider-liveramp.md | 6 +++--- .../plans/2026-11-02-identity-provider-uid2.md | 4 ++-- trusted-server.example.toml | 13 +++++++++---- 9 files changed, 30 insertions(+), 21 deletions(-) diff --git a/crates/trusted-server-identity-providers/src/common.rs b/crates/trusted-server-identity-providers/src/common.rs index 9b3ca954f..9495652b7 100644 --- a/crates/trusted-server-identity-providers/src/common.rs +++ b/crates/trusted-server-identity-providers/src/common.rs @@ -150,8 +150,12 @@ pub(crate) fn cookie_capabilities( Ok((capabilities, source.clone())) } -/// Whether a cookie-carried ID is worth storing: not blank and not an -/// opt-out placeholder such as `0` or an all-zero UUID. +/// Whether a cookie-carried ID is worth storing: not blank and not a +/// placeholder such as `0` or an all-zero UUID. +/// +/// A placeholder reads as "no ID", not as an opt-out: no vendor read with +/// this check documents it as its opt-out signal. Providers whose vendor +/// does (ID5's `"0"`) check for it separately and report an opt-out. #[must_use] pub(crate) fn is_usable_id(id: &str) -> bool { let id = id.trim(); diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 092626f41..41e56237c 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -952,7 +952,7 @@ the EC row when the cookies hold none; no cookie is set. | `kind` | String | Yes | Provider implementation; must be one this build includes | | `enabled` | Boolean | No | Whether the provider runs (default `true`) | | `endpoint` | String | No | HTTPS lookup URL, for kinds that make a lookup; its host must be in `allowed_domains` | -| `allowed_domains` | Array | No | Hosts the provider may call; needed with `endpoint` | +| `allowed_domains` | Array | No | Hosts the provider may call; required with an `endpoint` | | `sources` | Array | Yes | Source domains the provider owns; each needs an `[[ec.partners]]` entry | | `tcf_vendor_id` | Integer | No | TCF vendor ID that must have vendor consent | | `collect` | String | No | `"page"` (default) or `"after_response"`; see below | diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-connectid.md b/docs/superpowers/plans/2026-11-02-identity-provider-connectid.md index dc8e4a37f..2d70da7dd 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-connectid.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-connectid.md @@ -22,11 +22,11 @@ No Lockr capability: Lockr AIM is not documented to deliver ConnectID, so a `[se ## Stored ID - One EID for the single configured source, one UID with `atype = 3`. -- Expiry `ProviderExpiry::At(lastSynced + min(ttl, 14 days))`; a missing `ttl` uses the shared default TTL. The provider reads no clock: the coordinator's validation drops the ID once that time has passed. +- Expiry `ProviderExpiry::At(lastSynced + min(ttl, 14 days))`; as in Prebid, a missing or zero `ttl` means 14 days, a missing `lastSynced` is stored as `ProviderExpiry::After(14 days)`, and a lowercase `connectid` key is accepted. The provider checks expiry with the clock so an expired cookie does not hide a later live one; the coordinator's validation also drops the ID once that time has passed. ## Not ported from the reference provider -- The reference calls Yahoo's sync endpoint, which needs a hashed email on every call. The logged-in experience is out of scope, so nothing is ported; this provider reads the `connectId` cookie only. +- The reference calls Yahoo's sync endpoint, which needs only `pixelId` (`he` is optional), but its useful matches come from the hashed email. The logged-in experience is out of scope, so nothing is ported; this provider reads the `connectId` cookie only. ## Tasks diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-criteo.md b/docs/superpowers/plans/2026-11-02-identity-provider-criteo.md index d0f07847a..4bc43121a 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-criteo.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-criteo.md @@ -12,8 +12,8 @@ ## Cookie -- Prebid's `criteoIdSystem` keeps Criteo's bid ID in the `cto_bidid` cookie (and local storage) and sends that value unchanged, still percent-encoded, as the `criteo.com` EID. `cto_bundle` is Criteo's own sync state and is not an ID. -- Default `cookie_names = ["cto_bidid"]`, read in order. The value is stored verbatim, not decoded. +- Prebid's `criteoIdSystem` keeps Criteo's bid ID in the `cto_bidid` cookie (and local storage) and sends the `decodeURIComponent`-decoded value as the `criteo.com` EID (unwrapping a `{"criteoId":…}` value). `cto_bundle` is Criteo's own sync state and is not an ID. +- Default `cookie_names = ["cto_bidid"]`, read in order. The value is percent-decoded and stored as Prebid sends it. - A blank or opt-out value reads as no ID. Criteo does not set `cto_bidid` on every page view, so the ID arrives on a later request. No Lockr capability. @@ -30,6 +30,6 @@ No Lockr capability. ## Tasks - [ ] Add `criteo.rs`; register the kind in `is_registered_kind` and `build`. -- [ ] Unit tests: value stored verbatim, missing and blank cookies ignored. +- [ ] Unit tests: value stored decoded, `criteoId` object unwrapped, missing and blank cookies ignored. - [ ] Add a commented `kind = "criteo"` example to `trusted-server.example.toml`. - [ ] Run `cargo test-fastly` and `cargo clippy-fastly`. diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-id5.md b/docs/superpowers/plans/2026-11-02-identity-provider-id5.md index 3f0429360..dcf6d7147 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-id5.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-id5.md @@ -16,12 +16,12 @@ - No private provider state, so ID5's `signature` is not sent back. - No page URL in `LookupInput`, so `tml` is the publisher origin. - Endpoint, sources and the TCF vendor ID come from the common provider config, not hard-coded defaults. -- The legacy `universal_uid` fallback goes to the first configured source. An opted-out `universal_uid` of `0` is reported as `LookupOutcome::NoMatch`; there is no separate invalidation outcome. +- The legacy `universal_uid` fallback goes to the first configured source. An opted-out `universal_uid` of `0` is reported as `LookupOutcome::OptedOut`, detected on the reply as sent (not percent-decoded). ## Request - `POST /.json`, with `Content-Type: application/json` and `Origin: https://`. -- Body: `partner`, `v`, `o = "trusted-server"`, `nbPage = 1`, `gdpr`, and when present `gdpr_consent`, `us_privacy`, `gpp_string`, `gpp_sid` (comma-joined), `ua`, and `ip` (only when routable, see `common::is_routable_ip`). +- Body: `partner`, `v`, `o = "trusted-server"`, `nbPage = 1`, `gdpr`, and when present `gdpr_consent`, `us_privacy`, `gpp_string`, `gpp_sid` (comma-joined), `ua`, and `ip` (IPv4) or `ipv6` (only when routable, see `common::is_routable_ip`). - Confirm the `ip` and `ua` field names with ID5 for each partner account. ## Response diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md b/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md index 4af83fefb..d220dd019 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md @@ -16,7 +16,7 @@ - The attribute-to-source table is fixed, as in the reference provider; `settings.resolve` picks attributes from it, and every picked attribute's source must be one of the provider's `sources`. - No page URL in `LookupInput`, so `pu` is the publisher origin. - EID-level fields are preserved. As in Prebid, every attribute other than `nonId` is a partner ID whose UID carries `ext` `{"provider":"liveintent.com"}`; the Trade Desk ID's UID also carries `rtiPartner: "TDID"` and its EID `matcher = "liveintent.com"`. The reply's `segments` become the `nonId` EID's `ext` (`{"segments": ...}`), which needs the PR A change letting a provider return an owned EID `ext`. -- The `Expires` header becomes `ProviderExpiry::At`; the provider never reads the clock. +- The `Expires` header becomes `ProviderExpiry::After(Expires - Date)`; the provider never reads the clock. ## Settings @@ -37,7 +37,7 @@ No Lockr capability: Lockr AIM is not documented to deliver LiveIntent IDs, and ## Response - `204 No Content` is `LookupOutcome::NoMatch`. LiveIntent answers `204` to anonymous `gdpr=1` lookups. On the `prebid` path it returns only `nonId`; with the publisher's own `distributor_id` (for example `did-0123`) and the `_lc2_fpi` first-party ID as `duid`, it also returns partner IDs without its third-party cookie, so a publisher wanting partner IDs must set `distributor_id`; another non-2xx status or a non-object body is `ProviderError::InvalidResponse`. -- Each resolved attribute with a non-empty string value becomes one EID with `atype = 3`, borrowed from the body; `segments`, the TDID `matcher` and the UID `ext` values are added as above. +- Each resolved attribute with a non-empty string value becomes one EID with `atype = 3` (the resolved TDID gets `atype = 1`, as in Prebid), borrowed from the body; `segments`, the TDID `matcher` and the UID `ext` values are added as above. Segments whose wrapped `ext` exceeds `MAX_EXT_BYTES` are left out so the `nonId` EID is kept. ## Tasks diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md b/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md index 22e9c07a3..3b30ea7a8 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md @@ -4,9 +4,9 @@ **Status:** Implemented; checkboxes are not updated. -**Goal:** Store the LiveRamp RampID envelope the browser already sends in LiveRamp's own first-party cookie, as an `[[ec.identity_providers]]` entry with `kind = "liveramp"`. No login and no server call. +**Goal:** Store the LiveRamp RampID envelope the browser already sends in LiveRamp's own first-party cookie, as an `[[ec.identity_providers]]` entry with `kind = "liveramp"`. No login. With an `endpoint`, a visitor without an envelope cookie is also looked up server to server. -**Architecture:** `crates/trusted-server-identity-providers/src/liveramp.rs` holds `LiveRamp` (a `ProviderKind`), `LiveRampSettings` and `LiveRampProvider`. The provider implements only `IdentityProvider::read_request_cookies`; `build_request` always fails, so the coordinator never sends a lookup. The coordinator reads cookies on every navigation (a cookie repeating the held UID is not rewritten) and applies consent, validation, expiry and persistence exactly as for lookup results. +**Architecture:** `crates/trusted-server-identity-providers/src/liveramp.rs` holds `LiveRamp` (a `ProviderKind`), `LiveRampSettings` and `LiveRampProvider`. The provider reads the envelope cookies in `IdentityProvider::read_request_cookies`; without an `endpoint`, `build_request` fails, so no lookup is sent. The coordinator reads cookies on every navigation (a cookie repeating the held UID is not rewritten) and applies consent, validation, expiry and persistence exactly as for lookup results. **Spec:** `docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md`, section 7. @@ -21,7 +21,7 @@ - One EID for the single configured source, one UID with `atype = 3`. - Expiry `ProviderExpiry::After(15 days)` through `bounded_expiry`. -- Optional server lookup (with `endpoint`): `GET /api/identity/v2/envelope` with `pid` (`placement_id`), `it=15`, `iv=:`, `atype=2`, consent as `ct`/`cv` (TCF `4`, else US Privacy `3`), `gpp` and `gpp_sid`; headers `Origin` (publisher) and `X-Forwarded-For` (routable visitor IP, else no lookup). `200 {"envelope"}` is stored like the cookie; `204` is an opt-out; a blank envelope is no match; other statuses fail. Without `endpoint`, no server call. LiveRamp must enable Custom ID for the placement and approve the origin. +- Optional server lookup (with `endpoint`): `GET /api/identity/v2/envelope` with `pid` (`placement_id`), `it=15`, `iv=:`, `atype=3` (server to server; `2` is the ATS API), consent as `ct`/`cv` (TCF `4`, else US Privacy `3`), `gpp` and `gpp_sid` (the first section ID only); headers `Origin` (publisher) and `X-Forwarded-For` (routable visitor IP, else no lookup). `200 {"envelopes":[{"type":19,"value":…}]}`: the `RampID` entry (`type` 19) is stored like the cookie, and a reply without one is no match; `204` is an opt-out; a blank envelope is no match; other statuses fail. Without `endpoint`, no server call. LiveRamp must enable Custom ID for the placement and approve the origin. ## Lockr diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md b/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md index 4461c9dff..b53fc27ae 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md @@ -21,7 +21,7 @@ - One EID for the single configured source, one UID with `atype = 3`. - From the identity cookie: `ProviderExpiry::At(identity_expires)`; the coordinator's validation drops an expired token. - From the raw token cookie: `bounded_expiry(None)`, the shared default TTL. -- Refresh state `from = refresh_from`; the source becomes due then, and is refreshed only when the cookies no longer yield a usable token. A refresh is due only while the token is still live, so a failing refresh stops at its expiry; `rate_limit_per_hour` also bounds it. An `optout` reply removes the stored token. +- Refresh state `from = refresh_from`; the source becomes due then, and is refreshed once due, also while the cookie repeats the stored token (a cookie repeating the held token is not rewritten, so a failed refresh keeps the held refresh state). A refresh is due only while the token is still live, so a failing refresh stops at its expiry; `rate_limit_per_hour` also bounds it. An `optout` reply removes the stored token. ## Lockr @@ -31,7 +31,7 @@ - Token generation (needs a hashed email; out of scope). -Risk: a server refresh spends the SDK cookie's refresh token without rewriting `__uid_2`, so the browser SDK's own refresh of that identity may then fail. That is acceptable because server refresh runs only when the browser sends no usable token. +Risk: a server refresh does not rewrite `__uid_2`. Refreshing does not invalidate the previous token or refresh token, so the server refresh and the browser SDK's own refresh both succeed and fork into two token chains. A later navigation whose cookie still carries the older token then replaces the refreshed row token; `rate_limit_per_hour` bounds the resulting refreshes. ## Tasks diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 0c374a371..e37b65404 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -158,7 +158,8 @@ pull_sync_concurrency = 3 # ID5 (kind "id5"): anonymous lookup, no secrets. The first source receives # the legacy `universal_uid`. Use the endpoint and sources ID5 gives you. # Before any lookup it reads the first-party cookies in `cookie_names`, in -# order (Prebid's ID5 module writes `id5id`). +# order. Prebid's ID5 module writes the `id5id` cookie only with +# `storage.type: 'cookie'`; with `html5` storage the lookup runs. # # Lockr capability (id5, liveramp and uid2 only): Lockr AIM writes other # vendors' IDs into first-party cookies under key names configured per @@ -254,8 +255,11 @@ pull_sync_concurrency = 3 # # UID2 (kind "uid2"): reads the advertising token from the UID2 SDK's # `__uid_2` cookie until its `identity_expires`, falling back to Prebid's raw -# `__uid2_advertising_token` cookie. Takes exactly one source. `cookie_names` -# lists the raw token cookies, read in order after `identity_cookie_name`. +# `__uid2_advertising_token` cookie. Both clients keep the token in +# localStorage by default, which the server never sees: set the SDK's +# `useCookie: true` or Prebid's `params.storage: 'cookie'`. Takes exactly one +# source. `cookie_names` lists the raw token cookies, read in order after +# `identity_cookie_name`. # Supports the Lockr capability; Lockr cookies are read last, as raw tokens. # Optional server refresh: set `endpoint` to the UID2 operator's # `/v2/token/refresh` URL. The `__uid_2` cookie's refresh token and response @@ -301,7 +305,8 @@ pull_sync_concurrency = 3 # # Criteo (kind "criteo"): no lookup and no endpoint. Reads Criteo's bid ID # from the first-party `cto_bidid` cookie Prebid's Criteo module keeps and -# stores it verbatim as a device-based (atype 1) ID. Takes exactly one source. +# stores it percent-decoded, as Prebid sends it, as a device-based (atype 1) +# ID. Takes exactly one source. # [[ec.identity_providers]] # id = "criteo" # kind = "criteo" From 7e3dfc6a1771211a0dfa3c769b9d159c2efcaa1d Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 16:57:00 +0530 Subject: [PATCH 49/72] Reconcile the vendor providers with the updated collection core Carry the request cookies through the after-response lookup path, require a consented row in the coordinator tests, and update the provider tests for the owner-scoped OwnedEntries view and the partner-aware after-response builder. Signed-off-by: dhruv8sh --- .../trusted-server-adapter-fastly/src/app.rs | 2 +- .../src/ec/coordinator/mod.rs | 40 +++++++++---------- .../src/ec/coordinator/validate.rs | 3 ++ .../src/id5.rs | 4 +- .../src/lib.rs | 10 +++-- .../src/liveintent.rs | 6 +-- .../src/liveramp.rs | 10 ++--- .../src/sharedid.rs | 2 +- .../src/uid2.rs | 6 ++- 9 files changed, 45 insertions(+), 38 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index c811cc0de..6b0fb7915 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -1647,7 +1647,7 @@ mod tests { use trusted_server_core::ec::provider::{ EidLevelFields, EidShape, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderEid, ProviderError, ProviderExpiry, ProviderRequest, - ProviderResponse, ProviderUid, RequestCookies, SourceCapability, SourceInput, + ProviderResponse, ProviderUid, SourceCapability, SourceInput, }; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::TrustedServerError; diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 68ecf5800..4f1bba73d 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -1005,6 +1005,7 @@ mod tests { atype: None, ext: None, expiry: ProviderExpiry::At(UnixSeconds::new(5000)), + refresh: None, }], }]) } @@ -1183,8 +1184,8 @@ mod tests { ) } - fn run_with_cookies( - coordinator: &ProviderCoordinator, + fn run_with_cookies( + coordinator: &ProviderCoordinator

, transport: &CannedTransport, snapshot: &EcKvSnapshot, consent: &ConsentContext, @@ -1551,8 +1552,8 @@ mod tests { .expect("should build coordinator"); let transport = CannedTransport::new(ok_reply); - let gdpr = run(&coordinator, &transport, &missing(), &tcf_consent(vec![])); - let outside = run(&coordinator, &transport, &missing(), &granting()); + let gdpr = run(&coordinator, &transport, &live_row(), &tcf_consent(vec![])); + let outside = run(&coordinator, &transport, &live_row(), &granting()); let lookup = gdpr.iter().next().expect("should report the provider"); assert_eq!( @@ -1590,7 +1591,7 @@ mod tests { }; // Act - let results = run(&coordinator, &transport, &missing(), &consent); + let results = run(&coordinator, &transport, &live_row(), &consent); // Assert assert!( @@ -1700,7 +1701,7 @@ mod tests { )); let transport = CannedTransport::new(ok_reply); - let results = run(&coordinator, &transport, &missing(), &granting()); + let results = run(&coordinator, &transport, &live_row(), &granting()); let lookup = results.iter().next().expect("should report the provider"); assert!( @@ -1736,7 +1737,7 @@ mod tests { let coordinator = cookie_coordinator(); let transport = CannedTransport::new(ok_reply); - for snapshot in [missing(), held_snapshot("p-1")] { + for snapshot in [live_row(), held_snapshot("p-1")] { let results = run_with_cookies( &coordinator, &transport, @@ -1804,7 +1805,7 @@ mod tests { )); let transport = CannedTransport::new(ok_reply); - let results = run(&coordinator, &transport, &missing(), &granting()); + let results = run(&coordinator, &transport, &live_row(), &granting()); assert!( matches!(only_status(&results), LookupStatus::Found { kept: 1, .. }), @@ -1821,7 +1822,7 @@ mod tests { )); let transport = CannedTransport::new(ok_reply); - let results = run(&coordinator, &transport, &missing(), &granting()); + let results = run(&coordinator, &transport, &live_row(), &tcf_consent(vec![])); assert_eq!( only_status(&results), @@ -2421,13 +2422,10 @@ mod tests { #[test] fn a_provider_making_no_lookup_spends_no_rate_limit() { let coordinator = ProviderCoordinator::new( - vec![ - RegisteredProvider::new( - provider_config(""), - NoLookupProvider(echo_provider(SourceInput::Anonymous)), - ) - .with_rate_limit(NonZeroU32::new(1).expect("should be non-zero")), - ], + vec![RegisteredProvider::new( + provider_config("rate_limit_per_hour = 1"), + NoLookupProvider(echo_provider(SourceInput::Anonymous)), + )], partners(), Duration::from_millis(200), ) @@ -2435,12 +2433,12 @@ mod tests { let transport = CannedTransport::new(ok_reply); let limiter = CountingLimiter::default(); - let results = run_with(&coordinator, &transport, &missing(), &granting(), &limiter); + let results = run_with(&coordinator, &transport, &live_row(), &granting(), &limiter); assert_eq!( only_status(&results), - LookupStatus::Skipped(SkipReason::Input), - "should skip a provider that builds no request" + LookupStatus::RequestFailed, + "should report a provider that builds no request" ); assert_eq!(limiter.0.get(), 0, "should not check the rate limit"); } @@ -2527,7 +2525,7 @@ mod tests { let coordinator = two_source_coordinator(|provider| provider); let transport = CannedTransport::new(ok_reply); - let results = run(&coordinator, &transport, &missing(), &granting()); + let results = run(&coordinator, &transport, &live_row(), &granting()); assert_eq!( *transport.sent.borrow(), @@ -2680,7 +2678,7 @@ mod tests { let transport = CannedTransport::new(ok_reply); // Act - let results = run(&coordinator, &transport, &missing(), &tcf_consent(vec![])); + let results = run(&coordinator, &transport, &live_row(), &tcf_consent(vec![])); // Assert let lookup = results.iter().next().expect("should report the provider"); diff --git a/crates/trusted-server-core/src/ec/coordinator/validate.rs b/crates/trusted-server-core/src/ec/coordinator/validate.rs index 92203027e..d5cc50e95 100644 --- a/crates/trusted-server-core/src/ec/coordinator/validate.rs +++ b/crates/trusted-server-core/src/ec/coordinator/validate.rs @@ -317,6 +317,9 @@ mod tests { "should keep only usable refresh state" ); } + } + + #[test] fn clamps_absolute_expiry_to_thirty_days() { let partners = registry(); let provider = ProviderId::new("example-provider"); diff --git a/crates/trusted-server-identity-providers/src/id5.rs b/crates/trusted-server-identity-providers/src/id5.rs index 519b70427..e8d247301 100644 --- a/crates/trusted-server-identity-providers/src/id5.rs +++ b/crates/trusted-server-identity-providers/src/id5.rs @@ -523,7 +523,7 @@ mod tests { }, publisher_domain: "www.example.com", due_sources: &[], - current: OwnedEntries::new(None, &[]), + current: OwnedEntries::empty(), cookies: RequestCookies::default(), hashed_email: None, }; @@ -574,7 +574,7 @@ mod tests { }, publisher_domain: "www.example.com", due_sources: &[], - current: OwnedEntries::new(None, &[]), + current: OwnedEntries::empty(), cookies: RequestCookies::default(), hashed_email: None, }; diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index 21114d104..0db3138c8 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -228,6 +228,10 @@ mod tests { use trusted_server_core::ec::provider::validate_enabled_configs; use trusted_server_core::settings::IdentityProviderConfig; + fn partners_for(settings: &Settings) -> Arc { + Arc::new(partner_registry(&settings.ec).expect("should build partner registry")) + } + fn create_test_settings() -> Settings { Settings::from_toml( r#" @@ -497,7 +501,7 @@ mod tests { ]; let page = build_coordinator(&settings).expect("should build page coordinator"); - let after = build_after_response_coordinator(&settings) + let after = build_after_response_coordinator(&settings, partners_for(&settings)) .expect("should build after-response coordinator"); assert!( @@ -517,7 +521,7 @@ mod tests { let page = build_coordinator(&settings) .expect("should build page coordinator") .expect("should have a page provider"); - let after = build_after_response_coordinator(&settings) + let after = build_after_response_coordinator(&settings, partners_for(&settings)) .expect("should build after-response coordinator") .expect("should have an after-response provider"); @@ -530,7 +534,7 @@ mod tests { let settings = settings_with_cookie_providers("after_response", "after_response"); let page = build_coordinator(&settings).expect("should build page coordinator"); - let after = build_after_response_coordinator(&settings) + let after = build_after_response_coordinator(&settings, partners_for(&settings)) .expect("should build after-response coordinator"); assert!(page.is_none(), "should build no page coordinator"); diff --git a/crates/trusted-server-identity-providers/src/liveintent.rs b/crates/trusted-server-identity-providers/src/liveintent.rs index cade56c94..898ee327c 100644 --- a/crates/trusted-server-identity-providers/src/liveintent.rs +++ b/crates/trusted-server-identity-providers/src/liveintent.rs @@ -488,7 +488,7 @@ mod tests { }, publisher_domain: "www.example.com", due_sources: &due, - current: OwnedEntries::new(None, &[]), + current: OwnedEntries::empty(), cookies, hashed_email: None, }; @@ -529,7 +529,7 @@ mod tests { }, publisher_domain: "www.example.com", due_sources: &due[..1], - current: OwnedEntries::new(None, &[]), + current: OwnedEntries::empty(), cookies: RequestCookies::default(), hashed_email: None, }; @@ -752,7 +752,7 @@ mod tests { }, publisher_domain: "www.example.com", due_sources: &due, - current: OwnedEntries::new(None, &[]), + current: OwnedEntries::empty(), cookies: RequestCookies::default(), hashed_email: None, }; diff --git a/crates/trusted-server-identity-providers/src/liveramp.rs b/crates/trusted-server-identity-providers/src/liveramp.rs index ab1f66f57..53ab737a4 100644 --- a/crates/trusted-server-identity-providers/src/liveramp.rs +++ b/crates/trusted-server-identity-providers/src/liveramp.rs @@ -330,7 +330,7 @@ mod tests { use serde_json::json; use trusted_server_core::consent::types::ConsentContext; use trusted_server_core::ec::eid_types::{EidValidationError, UidValue}; - use trusted_server_core::ec::kv_types::MAX_UID_LENGTH; + use trusted_server_core::ec::kv_types::MAX_STORED_UID_LENGTH; use trusted_server_core::ec::provider::{ ClientSignals, OwnedEntries, ProviderExpiry, build_kind, }; @@ -391,7 +391,7 @@ mod tests { fn collects_envelopes_up_to_the_uid_length_limit_and_drops_longer_ones() { let provider = provider(&json!({})); - for len in [703, 1000, MAX_UID_LENGTH] { + for len in [703, 1000, MAX_STORED_UID_LENGTH] { let raw = "A".repeat(len); let ats = encoded(&json!({ "envelope": raw }), &STANDARD); for header in [format!("idl_env={raw}"), format!("_lr_env={ats}")] { @@ -407,7 +407,7 @@ mod tests { let (id, _) = read( &*provider, - &format!("idl_env={}", "A".repeat(MAX_UID_LENGTH + 1)), + &format!("idl_env={}", "A".repeat(MAX_STORED_UID_LENGTH + 1)), ) .expect("should read the oversized envelope"); assert!( @@ -566,7 +566,7 @@ mod tests { }, publisher_domain: "www.example.com", due_sources: &[], - current: OwnedEntries::new(None, &[]), + current: OwnedEntries::empty(), cookies: RequestCookies::default(), hashed_email: None, }; @@ -599,7 +599,7 @@ mod tests { }, publisher_domain: "www.example.com", due_sources: &[], - current: OwnedEntries::new(None, &[]), + current: OwnedEntries::empty(), cookies: RequestCookies::default(), hashed_email: None, } diff --git a/crates/trusted-server-identity-providers/src/sharedid.rs b/crates/trusted-server-identity-providers/src/sharedid.rs index cba5f75fd..f91e3d07b 100644 --- a/crates/trusted-server-identity-providers/src/sharedid.rs +++ b/crates/trusted-server-identity-providers/src/sharedid.rs @@ -219,7 +219,7 @@ mod tests { }, publisher_domain: "www.example.com", due_sources: &[], - current: trusted_server_core::ec::provider::OwnedEntries::new(None, &[]), + current: trusted_server_core::ec::provider::OwnedEntries::empty(), cookies: RequestCookies::default(), hashed_email: None, }; diff --git a/crates/trusted-server-identity-providers/src/uid2.rs b/crates/trusted-server-identity-providers/src/uid2.rs index aa25ca758..bf094c559 100644 --- a/crates/trusted-server-identity-providers/src/uid2.rs +++ b/crates/trusted-server-identity-providers/src/uid2.rs @@ -785,6 +785,7 @@ mod tests { fn refresh_posts_the_stored_refresh_token() { let entry = stored_entry(); let sources = [SourceDomain::try_from("uid2.example.com").expect("should build source")]; + let owner = ProviderId::new("uid2"); let consent = ConsentContext::default(); let lookup = LookupInput { ec_id: "ec-example", @@ -795,7 +796,7 @@ mod tests { }, publisher_domain: "www.example.com", due_sources: &sources, - current: OwnedEntries::new(Some(&entry), &sources), + current: OwnedEntries::new(Some(&entry), &sources, &owner), cookies: RequestCookies::default(), hashed_email: None, }; @@ -826,6 +827,7 @@ mod tests { fn makes_no_request_without_an_endpoint() { let entry = stored_entry(); let sources = [SourceDomain::try_from("uid2.example.com").expect("should build source")]; + let owner = ProviderId::new("uid2"); let consent = ConsentContext::default(); let lookup = LookupInput { ec_id: "ec-example", @@ -836,7 +838,7 @@ mod tests { }, publisher_domain: "www.example.com", due_sources: &sources, - current: OwnedEntries::new(Some(&entry), &sources), + current: OwnedEntries::new(Some(&entry), &sources, &owner), cookies: RequestCookies::default(), hashed_email: None, }; From 8db0bc90f969695fd659ec7176cfea99516abdde Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 17:01:03 +0530 Subject: [PATCH 50/72] Hand 204 lookup replies to the provider The coordinator now reports only a 404 as no match before parsing. A 204 reaches the provider, so LiveRamp's 204 opt-out clears its sources and is remembered, while ID5 and UID2 map their own 204 to no match. Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 49 +++++++++++++++++-- crates/trusted-server-core/src/ec/provider.rs | 4 ++ .../src/id5.rs | 3 ++ .../src/uid2.rs | 3 ++ 4 files changed, 55 insertions(+), 4 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 4f1bba73d..d72b026c7 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -418,10 +418,9 @@ impl ProviderCoordinator

{ return self.lookup(provider, status, Vec::new()); } }; - if matches!( - response.status, - http::StatusCode::NOT_FOUND | http::StatusCode::NO_CONTENT - ) { + // Only 404 is a match-free reply for every provider; a 204 reaches + // the provider, because some vendors use it to report an opt-out. + if response.status == http::StatusCode::NOT_FOUND { return self.lookup(provider, LookupStatus::NoMatch, Vec::new()); } if !response.status.is_success() { @@ -963,6 +962,10 @@ mod tests { if response.context != Some(ECHO_CONTEXT) { return Err(Report::new(ProviderError::InvalidResponse)); } + // Like LiveRamp, the echo vendor reports an opt-out with a 204. + if response.status == http::StatusCode::NO_CONTENT { + return Ok(LookupOutcome::OptedOut); + } let id = std::str::from_utf8(response.body) .map_err(|_| Report::new(ProviderError::InvalidResponse))?; if id == "optout" { @@ -2196,6 +2199,44 @@ mod tests { ); } + fn no_content_reply() -> Result> { + Ok(reply(http::StatusCode::NO_CONTENT, b"")) + } + + #[test] + fn a_204_reply_reaches_the_provider() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(no_content_reply); + + let results = run(&coordinator, &transport, &live_row(), &granting()); + + let lookup = results.iter().next().expect("should report the provider"); + assert_eq!( + lookup.status, + LookupStatus::Skipped(SkipReason::OptedOut), + "should report the opt-out the provider reads from a 204" + ); + assert_eq!( + lookup.cleared, + [SourceDomain::try_from("ids.example.com").expect("should parse")], + "should clear the provider's sources" + ); + } + + #[test] + fn a_404_reply_is_no_match_without_parsing() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(not_found_reply); + + let results = run(&coordinator, &transport, &live_row(), &granting()); + + assert_eq!( + only_status(&results), + LookupStatus::NoMatch, + "should report a 404 as no match" + ); + } + /// An echo provider whose request can never be built. struct UnbuildableProvider(EchoProvider); diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs index a4e9934ec..bb520e75c 100644 --- a/crates/trusted-server-core/src/ec/provider.rs +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -50,6 +50,10 @@ pub trait IdentityProvider: Send + Sync { /// Parses a provider response, borrowing from its body. /// + /// A `404` is reported as no match before this is called; every other + /// status, `204` included, is handed to the provider, which decides + /// whether it means no match, an opt-out or an error. + /// /// # Errors /// /// Returns [`ProviderError::InvalidResponse`] when the body cannot be diff --git a/crates/trusted-server-identity-providers/src/id5.rs b/crates/trusted-server-identity-providers/src/id5.rs index e8d247301..b553dcb63 100644 --- a/crates/trusted-server-identity-providers/src/id5.rs +++ b/crates/trusted-server-identity-providers/src/id5.rs @@ -387,6 +387,9 @@ impl IdentityProvider for Id5Provider { &self, response: ProviderResponse<'body>, ) -> Result, Report> { + if response.status == http::StatusCode::NO_CONTENT { + return Ok(LookupOutcome::NoMatch); + } if !response.status.is_success() { return Err(Report::new(ProviderError::InvalidResponse) .attach(format!("ID5 returned status {}", response.status))); diff --git a/crates/trusted-server-identity-providers/src/uid2.rs b/crates/trusted-server-identity-providers/src/uid2.rs index bf094c559..6b964fed7 100644 --- a/crates/trusted-server-identity-providers/src/uid2.rs +++ b/crates/trusted-server-identity-providers/src/uid2.rs @@ -319,6 +319,9 @@ impl IdentityProvider for Uid2Provider { &self, response: ProviderResponse<'body>, ) -> Result, Report> { + if response.status == http::StatusCode::NO_CONTENT { + return Ok(LookupOutcome::NoMatch); + } let key = response.context.ok_or_else(|| { Report::new(ProviderError::InvalidResponse).attach("no uid2 response key") })?; From fb0539e9142a9bd3990d640790491bc9d8792b64 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 17:04:04 +0530 Subject: [PATCH 51/72] Pass the request time to identity providers LookupInput carries the coordinator's request time and RequestCookies carries it to cookie readers, so the UID2 and ConnectID expiry checks and ID5's ts field use the navigation's one clock reading. Provider tests now read cookies at a fixed time. Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 8 ++- crates/trusted-server-core/src/ec/provider.rs | 40 ++++++++++-- .../src/common.rs | 9 ++- .../src/connectid.rs | 28 +++++---- .../src/id5.rs | 14 +++-- .../src/liveintent.rs | 3 + .../src/liveramp.rs | 2 + .../src/sharedid.rs | 1 + .../src/uid2.rs | 61 +++++++++++-------- 9 files changed, 111 insertions(+), 55 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index d72b026c7..b5d0536f3 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -574,7 +574,10 @@ impl IdentityCoordinator for ProviderCoordinator

{ // The vendor's own first-party cookies are read on every // navigation, so an opt-out or a changed ID is noticed while the // sources still hold live UIDs. Their IDs stand in for a lookup. - match registered.provider.read_request_cookies(lookup.cookies) { + match registered + .provider + .read_request_cookies(lookup.cookies.at(lookup.now)) + { CookieRead::OptedOut => { log::debug!( "Identity lookup: provider={} opted out in its cookies", @@ -640,7 +643,8 @@ impl IdentityCoordinator for ProviderCoordinator

{ publisher_domain: lookup.publisher_domain, due_sources: &due, current: OwnedEntries::new(entry, ®istered.policy.sources, ®istered.id), - cookies: lookup.cookies, + cookies: lookup.cookies.at(lookup.now), + now: lookup.now, hashed_email: None, }; if let Some(mut eids) = registered.provider.generate(&input) { diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs index bb520e75c..930014d7d 100644 --- a/crates/trusted-server-core/src/ec/provider.rs +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -171,16 +171,27 @@ impl<'cookie> From>>> for CookieRead<'cookie> { /// The request's `Cookie` header, lent to providers that read their own /// first-party cookies. /// +/// It also carries the request time, so a provider judges a cookie's stated +/// expiry against the navigation's one clock reading rather than its own. /// Its `Debug` output names the cookies but never prints their values. -#[derive(Clone, Copy, Default)] -pub struct RequestCookies<'a>(&'a str); +#[derive(Clone, Copy)] +pub struct RequestCookies<'a> { + header: &'a str, + now: UnixSeconds, +} + +impl Default for RequestCookies<'_> { + fn default() -> Self { + Self::new("") + } +} impl core::fmt::Debug for RequestCookies<'_> { fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { formatter .debug_list() .entries( - self.0 + self.header .split(';') .filter_map(|pair| pair.split_once('=')) .map(|(name, _)| name.trim()), @@ -193,7 +204,24 @@ impl<'a> RequestCookies<'a> { /// Wraps a `Cookie` header value (pairs separated by `;`). #[must_use] pub fn new(header: &'a str) -> Self { - Self(header) + Self { + header, + now: UnixSeconds::new(0), + } + } + + /// The same cookies, read at request time `now`. + /// + /// The coordinator sets it before handing the cookies to a provider. + #[must_use] + pub fn at(self, now: UnixSeconds) -> Self { + Self { now, ..self } + } + + /// The request time the cookies are read at. + #[must_use] + pub fn now(&self) -> UnixSeconds { + self.now } /// Returns the first non-empty value of cookie `name`, undecoded and @@ -215,7 +243,7 @@ impl<'a> RequestCookies<'a> { where 'a: 'name, { - self.0 + self.header .split(';') .filter_map(|pair| pair.split_once('=')) .filter(move |(key, _)| key.trim() == name) @@ -289,6 +317,8 @@ pub struct LookupInput<'a> { /// The request's cookies, for a vendor first-party cookie a lookup /// sends as input. pub cookies: RequestCookies<'a>, + /// Request time; providers use it instead of reading the clock. + pub now: UnixSeconds, /// Always `None` in V1. pub hashed_email: Option<&'a HashedEmail>, } diff --git a/crates/trusted-server-identity-providers/src/common.rs b/crates/trusted-server-identity-providers/src/common.rs index 9495652b7..2766500fa 100644 --- a/crates/trusted-server-identity-providers/src/common.rs +++ b/crates/trusted-server-identity-providers/src/common.rs @@ -8,8 +8,7 @@ use std::time::Duration; use error_stack::Report; use serde::Deserialize; use serde_json::value::RawValue; -use trusted_server_core::ec::current_timestamp; -use trusted_server_core::ec::eid_types::SourceDomain; +use trusted_server_core::ec::eid_types::{SourceDomain, UnixSeconds}; use trusted_server_core::ec::provider::{ EidLevelFields, EidShape, ProviderCapabilities, ProviderCommon, ProviderConfigError, ProviderEid, ProviderExpiry, ProviderKind, ProviderUid, RequestCookies, SourceCapability, @@ -335,11 +334,11 @@ impl CookieReader { } } -/// Whether a cookie's stated expiry has already passed. +/// Whether a cookie's stated expiry has passed at request time `now`. /// /// Cookie readers skip such a value so it never hides a later live one. -pub(crate) fn is_expired(expiry: &ProviderExpiry) -> bool { - matches!(expiry, ProviderExpiry::At(at) if at.get() <= current_timestamp()) +pub(crate) fn is_expired(expiry: &ProviderExpiry, now: UnixSeconds) -> bool { + matches!(expiry, ProviderExpiry::At(at) if *at <= now) } /// A complete `OpenRTB` EID as a vendor returns it, borrowed from the body. diff --git a/crates/trusted-server-identity-providers/src/connectid.rs b/crates/trusted-server-identity-providers/src/connectid.rs index f7441efa3..e33fe75cd 100644 --- a/crates/trusted-server-identity-providers/src/connectid.rs +++ b/crates/trusted-server-identity-providers/src/connectid.rs @@ -106,7 +106,7 @@ fn from_millis(ms: f64) -> Duration { } /// Decodes one cookie value into the ID and its expiry. -fn connect_id(value: &str) -> Option<(String, ProviderExpiry)> { +fn connect_id(value: &str, now: UnixSeconds) -> Option<(String, ProviderExpiry)> { let decoded = percent_decoded(value); let cookie: ConnectIdCookie = serde_json::from_str(&decoded).ok()?; if !is_usable_id(&cookie.connect_id) { @@ -123,7 +123,7 @@ fn connect_id(value: &str) -> Option<(String, ProviderExpiry)> { let expires = from_millis(last_synced_ms).saturating_add(ttl); let expiry = ProviderExpiry::At(UnixSeconds::new(expires.as_secs())); // An expired cookie never hides a later live one. - (!is_expired(&expiry)).then_some((cookie.connect_id, expiry)) + (!is_expired(&expiry, now)).then_some((cookie.connect_id, expiry)) } impl IdentityProvider for ConnectIdProvider { @@ -149,7 +149,10 @@ impl IdentityProvider for ConnectIdProvider { &self, cookies: RequestCookies<'cookie>, ) -> CookieRead<'cookie> { - let Some((id, expiry)) = self.cookies.read(cookies, connect_id) else { + let Some((id, expiry)) = self + .cookies + .read(cookies, |value| connect_id(value, cookies.now())) + else { return CookieRead::Absent; }; CookieRead::Found(vec![cookie_eid(&self.source, Cow::Owned(id), expiry)]) @@ -164,6 +167,9 @@ mod tests { use super::*; + /// Request time the tests read cookies at. + const NOW: UnixSeconds = UnixSeconds::new(1_700_000_000); + fn provider() -> Box { let config: IdentityProviderConfig = serde_json::from_value(json!({ "id": "connectid", @@ -178,7 +184,7 @@ mod tests { fn read(cookie: &serde_json::Value) -> Option<(String, ProviderExpiry)> { let header = format!("connectId={}", urlencoding::encode(&cookie.to_string())); let eids = provider() - .read_request_cookies(RequestCookies::new(&header)) + .read_request_cookies(RequestCookies::new(&header).at(NOW)) .into_found()?; assert_eq!( eids[0].source, "connectid.example.com", @@ -202,7 +208,7 @@ mod tests { "connectId": "cid-example-1", "he": "example-hashed-email", "puid": "example-puid", - "lastSynced": 4_700_000_000_000_u64, + "lastSynced": 1_800_000_000_000_u64, "lastUsed": 4_700_000_100_000_u64, "ttl": 86_400_000, }); @@ -216,7 +222,7 @@ mod tests { ); assert_eq!( expires_at(read), - 4_700_086_400, + 1_800_086_400, "should expire at lastSynced + ttl" ); } @@ -224,7 +230,7 @@ mod tests { #[test] fn caps_the_ttl_and_defaults_a_missing_or_zero_one() { // A far-future sync keeps the cookie live whatever today's date is. - let synced_secs = 4_700_000_000_u64; + let synced_secs = 1_800_000_000_u64; let long = json!({ "connectId": "cid", "lastSynced": synced_secs * 1000, "ttl": 1e15 }); let missing = json!({ "connectId": "cid", "lastSynced": synced_secs * 1000 }); let zero = json!({ "connectId": "cid", "lastSynced": synced_secs * 1000, "ttl": 0 }); @@ -245,7 +251,7 @@ mod tests { #[test] fn reads_a_lowercase_key_and_a_missing_last_sync_as_fresh() { - let lowercase = json!({ "connectid": "cid", "lastSynced": 4_700_000_000_000_u64 }); + let lowercase = json!({ "connectid": "cid", "lastSynced": 1_800_000_000_000_u64 }); let unsynced = json!({ "connectId": "cid" }); assert_eq!( @@ -262,7 +268,7 @@ mod tests { #[test] fn an_expired_cookie_does_not_hide_a_later_live_one() { let expired = json!({ "connectId": "cid-old", "lastSynced": 1000, "ttl": 1000 }); - let live = json!({ "connectId": "cid-live", "lastSynced": 4_700_000_000_000_u64 }); + let live = json!({ "connectId": "cid-live", "lastSynced": 1_800_000_000_000_u64 }); let header = format!( "connectId={}; connectId={}", urlencoding::encode(&expired.to_string()), @@ -270,7 +276,7 @@ mod tests { ); let eids = provider() - .read_request_cookies(RequestCookies::new(&header)) + .read_request_cookies(RequestCookies::new(&header).at(NOW)) .into_found() .expect("should read the live cookie"); @@ -293,7 +299,7 @@ mod tests { } assert!( provider() - .read_request_cookies(RequestCookies::new("connectId={broken")) + .read_request_cookies(RequestCookies::new("connectId={broken").at(NOW)) .into_found() .is_none(), "should ignore malformed JSON" diff --git a/crates/trusted-server-identity-providers/src/id5.rs b/crates/trusted-server-identity-providers/src/id5.rs index b553dcb63..04727fe00 100644 --- a/crates/trusted-server-identity-providers/src/id5.rs +++ b/crates/trusted-server-identity-providers/src/id5.rs @@ -46,8 +46,7 @@ use error_stack::{Report, ResultExt as _}; use http::header::{CONTENT_TYPE, ORIGIN}; use http::{HeaderMap, HeaderValue, Method}; use serde::{Deserialize, Serialize}; -use trusted_server_core::ec::current_timestamp; -use trusted_server_core::ec::eid_types::SourceDomain; +use trusted_server_core::ec::eid_types::{SourceDomain, UnixSeconds}; use trusted_server_core::ec::provider::{ CookieRead, EidLevelFields, EidShape, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, ProviderConfigError, ProviderEid, ProviderError, @@ -63,9 +62,9 @@ use crate::common::{ is_usable_id, join_section_ids, percent_decoded, }; -/// Formats the current time as the RFC 3339 `ts` the `/gs/v2` fetch requires. -fn request_timestamp() -> Result> { - let now = i64::try_from(current_timestamp()).change_context(ProviderError::Request)?; +/// Formats the request time as the RFC 3339 `ts` the `/gs/v2` fetch requires. +fn request_timestamp(now: UnixSeconds) -> Result> { + let now = i64::try_from(now.get()).change_context(ProviderError::Request)?; DateTime::::from_timestamp(now, 0) .map(|time| time.to_rfc3339_opts(SecondsFormat::Secs, true)) .ok_or_else(|| Report::new(ProviderError::Request).attach("current time is out of range")) @@ -362,7 +361,7 @@ impl IdentityProvider for Id5Provider { gpp_string: consent.raw_gpp_string.as_deref(), gpp_sid: consent.gpp_section_ids.as_deref().map(join_section_ids), tml: &origin, - ts: request_timestamp()?, + ts: request_timestamp(lookup.now)?, ua: lookup.client.user_agent, ip: routable_ip.filter(IpAddr::is_ipv4).map(|ip| ip.to_string()), ipv6: routable_ip.filter(IpAddr::is_ipv6).map(|ip| ip.to_string()), @@ -529,6 +528,7 @@ mod tests { current: OwnedEntries::empty(), cookies: RequestCookies::default(), hashed_email: None, + now: trusted_server_core::ec::eid_types::UnixSeconds::new(1_700_000_000), }; let request = provider @@ -580,6 +580,7 @@ mod tests { current: OwnedEntries::empty(), cookies: RequestCookies::default(), hashed_email: None, + now: trusted_server_core::ec::eid_types::UnixSeconds::new(1_700_000_000), }; let request = provider @@ -607,6 +608,7 @@ mod tests { current: OwnedEntries::empty(), cookies: RequestCookies::default(), hashed_email: None, + now: trusted_server_core::ec::eid_types::UnixSeconds::new(1_700_000_000), }; let request = provider diff --git a/crates/trusted-server-identity-providers/src/liveintent.rs b/crates/trusted-server-identity-providers/src/liveintent.rs index 898ee327c..aeebb8723 100644 --- a/crates/trusted-server-identity-providers/src/liveintent.rs +++ b/crates/trusted-server-identity-providers/src/liveintent.rs @@ -491,6 +491,7 @@ mod tests { current: OwnedEntries::empty(), cookies, hashed_email: None, + now: trusted_server_core::ec::eid_types::UnixSeconds::new(1_700_000_000), }; for (header, expected) in [ @@ -532,6 +533,7 @@ mod tests { current: OwnedEntries::empty(), cookies: RequestCookies::default(), hashed_email: None, + now: trusted_server_core::ec::eid_types::UnixSeconds::new(1_700_000_000), }; let request = provider @@ -755,6 +757,7 @@ mod tests { current: OwnedEntries::empty(), cookies: RequestCookies::default(), hashed_email: None, + now: trusted_server_core::ec::eid_types::UnixSeconds::new(1_700_000_000), }; let request = provider diff --git a/crates/trusted-server-identity-providers/src/liveramp.rs b/crates/trusted-server-identity-providers/src/liveramp.rs index 53ab737a4..a40e0b05c 100644 --- a/crates/trusted-server-identity-providers/src/liveramp.rs +++ b/crates/trusted-server-identity-providers/src/liveramp.rs @@ -569,6 +569,7 @@ mod tests { current: OwnedEntries::empty(), cookies: RequestCookies::default(), hashed_email: None, + now: trusted_server_core::ec::eid_types::UnixSeconds::new(1_700_000_000), }; assert!( @@ -602,6 +603,7 @@ mod tests { current: OwnedEntries::empty(), cookies: RequestCookies::default(), hashed_email: None, + now: trusted_server_core::ec::eid_types::UnixSeconds::new(1_700_000_000), } } diff --git a/crates/trusted-server-identity-providers/src/sharedid.rs b/crates/trusted-server-identity-providers/src/sharedid.rs index f91e3d07b..62f9dc21d 100644 --- a/crates/trusted-server-identity-providers/src/sharedid.rs +++ b/crates/trusted-server-identity-providers/src/sharedid.rs @@ -222,6 +222,7 @@ mod tests { current: trusted_server_core::ec::provider::OwnedEntries::empty(), cookies: RequestCookies::default(), hashed_email: None, + now: trusted_server_core::ec::eid_types::UnixSeconds::new(1_700_000_000), }; provider.generate(&lookup) } diff --git a/crates/trusted-server-identity-providers/src/uid2.rs b/crates/trusted-server-identity-providers/src/uid2.rs index 6b964fed7..43bf3d561 100644 --- a/crates/trusted-server-identity-providers/src/uid2.rs +++ b/crates/trusted-server-identity-providers/src/uid2.rs @@ -365,15 +365,19 @@ impl IdentityProvider for Uid2Provider { let token = cookies .get_all(&self.identity_cookie_name) .filter_map(identity) - .find(|token| !is_expired(&token.expiry)) + .find(|token| !is_expired(&token.expiry, cookies.now())) .or_else(|| { self.cookies .read(cookies, |value| match token_cookie(value)? { - TokenCookie::Token(token, expiry) if !is_expired(&expiry) => Some(Token { - token, - expiry, - refresh: None, - }), + TokenCookie::Token(token, expiry) + if !is_expired(&expiry, cookies.now()) => + { + Some(Token { + token, + expiry, + refresh: None, + }) + } TokenCookie::Token(..) | TokenCookie::OptedOut => None, }) }); @@ -417,6 +421,9 @@ mod tests { use super::*; + /// Request time the tests read cookies at. + const NOW: UnixSeconds = UnixSeconds::new(1_700_000_000); + fn provider(settings: &serde_json::Value) -> Box { let config: IdentityProviderConfig = serde_json::from_value(json!({ "id": "uid2", @@ -430,7 +437,7 @@ mod tests { fn read(provider: &dyn IdentityProvider, header: &str) -> Option<(String, ProviderExpiry)> { let eids = provider - .read_request_cookies(RequestCookies::new(header)) + .read_request_cookies(RequestCookies::new(header).at(NOW)) .into_found()?; assert_eq!(eids[0].source, "uid2.example.com", "should use the source"); let uid = &eids[0].uids[0]; @@ -458,7 +465,7 @@ mod tests { let header = identity_cookie(&json!({ "advertising_token": "AgAAAAexampletoken+/=", "refresh_token": "example-refresh-token", - "identity_expires": 4_700_000_000_123_u64, + "identity_expires": 1_800_000_000_123_u64, "refresh_from": 4_699_990_000_000_u64, "refresh_expires": 4_702_000_000_000_u64, "refresh_response_key": "example-response-key", @@ -471,7 +478,7 @@ mod tests { "should store only the token" ); assert!( - matches!(expiry, ProviderExpiry::At(at) if at.get() == 4_700_000_000), + matches!(expiry, ProviderExpiry::At(at) if at.get() == 1_800_000_000), "should expire at identity_expires" ); } @@ -507,7 +514,7 @@ mod tests { "advertising_token": "AgAAAAlatesttoken", "refresh_token": "example-refresh-token", "refresh_response_key": "example-response-key", - "identity_expires": 4_700_000_000_456_u64, + "identity_expires": 1_800_000_000_456_u64, "refresh_from": 4_699_990_000_000_u64, "refresh_expires": 4_702_000_000_000_u64, }, @@ -520,7 +527,7 @@ mod tests { "should store only the latest advertising token" ); assert!( - matches!(expiry, ProviderExpiry::At(at) if at.get() == 4_700_000_000), + matches!(expiry, ProviderExpiry::At(at) if at.get() == 1_800_000_000), "should expire at the latest identity_expires" ); } @@ -530,7 +537,7 @@ mod tests { let provider = provider(&json!({})); let own = identity_cookie(&json!({ "advertising_token": "AgAAAAowntoken", - "identity_expires": 4_700_000_000_000_u64, + "identity_expires": 1_800_000_000_000_u64, })); let opt_out = prebid_cookie(&json!({ "originalToken": { "advertising_token": "AgAAAAoriginaltoken" }, @@ -539,7 +546,7 @@ mod tests { let header = format!("{own}; {opt_out}"); - let read = provider.read_request_cookies(RequestCookies::new(&header)); + let read = provider.read_request_cookies(RequestCookies::new(&header).at(NOW)); assert!( matches!(read, CookieRead::OptedOut), @@ -552,18 +559,18 @@ mod tests { let provider = provider(&json!({})); let live = identity_cookie(&json!({ "advertising_token": "AgAAAAowntoken", - "identity_expires": 4_700_000_000_000_u64, + "identity_expires": 1_800_000_000_000_u64, })); let opt_out = identity_cookie(&json!({ "status": "optout" })); let fallback = prebid_cookie(&json!({ "latestToken": { "advertising_token": "AgAAAAlatesttoken", - "identity_expires": 4_700_000_000_000_u64, + "identity_expires": 1_800_000_000_000_u64, }, })); let header = format!("{live}; {opt_out}; {fallback}"); - let read = provider.read_request_cookies(RequestCookies::new(&header)); + let read = provider.read_request_cookies(RequestCookies::new(&header).at(NOW)); assert!( matches!(read, CookieRead::OptedOut), @@ -577,13 +584,13 @@ mod tests { let token = prebid_cookie(&json!({ "latestToken": { "advertising_token": "AgAAAAlatesttoken", - "identity_expires": 4_700_000_000_000_u64, + "identity_expires": 1_800_000_000_000_u64, }, })); let opt_out = prebid_cookie(&json!({ "latestToken": "optout" })); let header = format!("{token}; {opt_out}"); - let read = provider.read_request_cookies(RequestCookies::new(&header)); + let read = provider.read_request_cookies(RequestCookies::new(&header).at(NOW)); assert!( matches!(read, CookieRead::OptedOut), @@ -601,7 +608,7 @@ mod tests { let live = prebid_cookie(&json!({ "latestToken": { "advertising_token": "AgAAAAlivetoken", - "identity_expires": 4_700_000_000_000_u64, + "identity_expires": 1_800_000_000_000_u64, }, })); let header = format!("{expired}; {live}"); @@ -627,7 +634,7 @@ mod tests { ] { assert!( matches!( - provider.read_request_cookies(RequestCookies::new(&header)), + provider.read_request_cookies(RequestCookies::new(&header).at(NOW)), CookieRead::Absent ), "should read nothing from {header:?}" @@ -658,7 +665,7 @@ mod tests { })); let own = identity_cookie(&json!({ "advertising_token": "AgAAAAowntoken", - "identity_expires": 4_700_000_000_000_u64, + "identity_expires": 1_800_000_000_000_u64, })); let both = read(&*enabled, &format!("example_lockr_uid2=AgAAAAlockr; {own}")); @@ -713,7 +720,7 @@ mod tests { identity_cookie(&json!({ "advertising_token": "AgAAAAexampletoken", "refresh_token": "example-refresh-token", - "identity_expires": 4_700_000_000_000_u64, + "identity_expires": 1_800_000_000_000_u64, "refresh_from": 4_699_990_000_000_u64, "refresh_expires": 4_702_000_000_000_u64, "refresh_response_key": BASE64.encode([7_u8; 32]), @@ -725,7 +732,7 @@ mod tests { header: &str, ) -> Option<(String, ProviderExpiry)> { let eids = provider - .read_request_cookies(RequestCookies::new(header)) + .read_request_cookies(RequestCookies::new(header).at(NOW)) .into_found() .expect("should read the token"); let refresh = eids[0].uids[0].refresh.as_ref()?; @@ -767,15 +774,15 @@ mod tests { AgentType::try_from(3).expect("should build atype"), None, Some(ProviderId::new("uid2")), - Expiry::At(UnixSeconds::new(4_700_000_000)), + Expiry::At(UnixSeconds::new(1_800_000_000)), ) .with_refresh(KvRefresh { state, - from: UnixSeconds::new(4_699_990_000), + from: UnixSeconds::new(1_799_990_000), }); let record = KvPartnerId::from_eids( vec![KvEid::new(EidFields::default(), vec![uid])], - UnixSeconds::new(4_699_995_000), + UnixSeconds::new(1_799_995_000), ) .expect("should keep the live uid"); let mut entry = KvEntry::tombstone(1); @@ -802,6 +809,7 @@ mod tests { current: OwnedEntries::new(Some(&entry), &sources, &owner), cookies: RequestCookies::default(), hashed_email: None, + now: trusted_server_core::ec::eid_types::UnixSeconds::new(1_700_000_000), }; let request = refreshing_provider() @@ -844,6 +852,7 @@ mod tests { current: OwnedEntries::new(Some(&entry), &sources, &owner), cookies: RequestCookies::default(), hashed_email: None, + now: trusted_server_core::ec::eid_types::UnixSeconds::new(1_700_000_000), }; assert!( From 5c0bca7451910c32f04a17dd02666a7f033aa621 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 17:06:48 +0530 Subject: [PATCH 52/72] Keep refresh state that is already due Validation dropped refresh credentials whose refresh time had passed, so a live UID2 identity imported after its refresh_from, or kept through a failed refresh, could never be refreshed again. Due state is now stored and the source stays due; rate_limit_per_hour bounds the retries. Signed-off-by: dhruv8sh --- .../src/ec/coordinator/validate.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/validate.rs b/crates/trusted-server-core/src/ec/coordinator/validate.rs index d5cc50e95..f0a5be90c 100644 --- a/crates/trusted-server-core/src/ec/coordinator/validate.rs +++ b/crates/trusted-server-core/src/ec/coordinator/validate.rs @@ -144,11 +144,13 @@ fn stored_uid( return Ok(stored); }; // Unusable refresh state is dropped, never the UID: a refresh with no - // stated time, with empty or oversized state, or due already (which - // would make the source due on every navigation) is simply not stored. + // stated time or with empty or oversized state is simply not stored. + // State that is already due is kept, so a live UID imported after its + // refresh time, or held through a failed refresh, can still be + // refreshed; `rate_limit_per_hour` bounds the retries. let usable = !refresh.state.is_empty() && refresh.state.len() <= MAX_REFRESH_STATE_BYTES; match resolve_expiry(refresh.from, policy.now) { - Some(from) if usable && from > policy.now => Ok(stored.with_refresh(KvRefresh { + Some(from) if usable => Ok(stored.with_refresh(KvRefresh { state: refresh.state.into_owned(), from, })), @@ -265,7 +267,7 @@ mod tests { } #[test] - fn keeps_the_uid_when_its_refresh_state_is_unusable() { + fn keeps_due_refresh_state_and_drops_only_unusable_state() { let partners = registry(); let provider = ProviderId::new("example-provider"); let claimed = claimed(); @@ -282,17 +284,17 @@ mod tests { ( "state".to_owned(), ProviderExpiry::After(Duration::ZERO), - None, + Some(UnixSeconds::new(1000)), ), ( "state".to_owned(), ProviderExpiry::At(UnixSeconds::new(1000)), - None, + Some(UnixSeconds::new(1000)), ), ( "state".to_owned(), ProviderExpiry::At(UnixSeconds::new(999)), - None, + Some(UnixSeconds::new(999)), ), ]; @@ -314,7 +316,7 @@ mod tests { .refresh() .map(|refresh| refresh.from), expected, - "should keep only usable refresh state" + "should keep usable refresh state, due or not" ); } } From 7667acf3cc4859233b7f09f391445d0a706bff9b Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 17:06:49 +0530 Subject: [PATCH 53/72] Keep a refreshed UID2 token over an older cookie token A server-side refresh updates the row but not the browser's cookies, so the next navigation's older __uid_2 token replaced the refreshed token and its refresh state. Providers can now ask the coordinator to keep a live stored UID whose absolute expiry outlives the cookie's; UID2 does. A cookie opt-out still wins, and a newer SDK token is still taken. Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 151 +++++++++++++++++- crates/trusted-server-core/src/ec/provider.rs | 17 ++ .../src/uid2.rs | 7 + 3 files changed, 168 insertions(+), 7 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index b5d0536f3..dfbe5d828 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -25,7 +25,7 @@ use crate::consent::jurisdiction::Jurisdiction; use crate::consent::{allows_eid_persistence, tcf_vendor_consent_denied}; use crate::ec::EcKvSnapshot; use crate::ec::consent::ec_consent_granted; -use crate::ec::eid_types::{EidFields, ProviderId, SourceDomain, UnixSeconds}; +use crate::ec::eid_types::{EidFields, Expiry, ProviderId, SourceDomain, UnixSeconds}; use crate::ec::generation::ec_hash; use crate::ec::kv_types::{KvEid, KvEntry, KvUid}; use crate::ec::provider::{ @@ -587,7 +587,15 @@ impl IdentityCoordinator for ProviderCoordinator

{ continue; } CookieRead::Found(eids) => { - let found = self.validated(provider, eids, lookup.now); + let mut found = self.validated(provider, eids, lookup.now); + // A cookie older than a server-side refresh does not + // roll the refreshed UID back. + if registered.provider.keeps_fresher_stored_uids() { + found.entries.retain(|(source, eids)| { + !stored_outlives_cookie(entry, source, eids, lookup.now) + }); + recount_found(&mut found); + } if matches!(found.status, LookupStatus::Found { .. }) { // Only the sources the cookies filled are spared a // lookup; the provider's other due sources are @@ -610,7 +618,6 @@ impl IdentityCoordinator for ProviderCoordinator

{ // on every navigation, and a failed lookup for a // still-due source would replace the held record and // its refresh state. - let mut found = found; found.entries.retain(|(source, eids)| { !holds_same_uids_for(entry, source, eids, lookup.now) }); @@ -848,6 +855,39 @@ fn recount_found(lookup: &mut ProviderLookup) { } } +/// Whether the row holds a live UID for `source` whose absolute expiry is +/// later than that of every UID in `eids`, so the cookie is older than the +/// stored record. +fn stored_outlives_cookie( + entry: Option<&KvEntry>, + source: &SourceDomain, + eids: &[KvEid], + now: UnixSeconds, +) -> bool { + let Some(record) = entry.and_then(|entry| entry.ids.get(source.as_str())) else { + return false; + }; + let Some(stored_until) = record + .eids() + .iter() + .flat_map(KvEid::uids) + .filter(|uid| uid.is_live(now)) + .filter_map(|uid| match uid.expiry() { + Expiry::At(at) => Some(at), + Expiry::Never => None, + }) + .max() + else { + return false; + }; + eids.iter() + .flat_map(KvEid::uids) + .all(|uid| match uid.expiry() { + Expiry::At(at) => at < stored_until, + Expiry::Never => false, + }) +} + /// Whether the row holds exactly the live UIDs in `eids` for `source`. /// /// UIDs are compared by ID, agent type, `ext` and their EID-level fields; @@ -913,7 +953,7 @@ mod tests { use super::*; use crate::consent::types::TcfConsent; use crate::ec::coordinator::results::DropCounts; - use crate::ec::eid_types::{AgentType, EidFields, Expiry, UidValue}; + use crate::ec::eid_types::{AgentType, UidValue}; use crate::ec::kv::KvIdentityGraph; use crate::ec::kv_types::{KvEid, KvPartnerId, KvRefresh, KvUid}; use crate::ec::provider::{ @@ -1136,9 +1176,9 @@ mod tests { ) } - fn coordinator_with( - registered: RegisteredProvider, - ) -> ProviderCoordinator { + fn coordinator_with( + registered: RegisteredProvider

, + ) -> ProviderCoordinator

{ ProviderCoordinator::new(vec![registered], partners(), Duration::from_millis(200)) .expect("should build coordinator") } @@ -2651,6 +2691,103 @@ mod tests { } } + /// A cookie provider that keeps a stored UID outliving the cookie's. + struct RefreshingCookieProvider(EchoProvider); + + impl IdentityProvider for RefreshingCookieProvider { + fn capabilities(&self) -> &ProviderCapabilities { + self.0.capabilities() + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + self.0.build_request(lookup) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + self.0.parse_response(response) + } + + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> CookieRead<'cookie> { + self.0.read_request_cookies(cookies) + } + + fn keeps_fresher_stored_uids(&self) -> bool { + true + } + } + + fn record_until(id: &str, until: u64) -> KvPartnerId { + let uid = KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(3).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::At(UnixSeconds::new(until)), + ); + KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![uid])], + UnixSeconds::new(1000), + ) + .expect("should build provider record") + } + + #[test] + fn an_older_cookie_does_not_roll_back_a_refreshed_uid() { + // Arrange: the cookie's UID expires at 5000; a refresh stored one + // living until 6000. + let coordinator = coordinator_with(RegisteredProvider::new( + provider_config(""), + RefreshingCookieProvider(cookie_provider(SourceInput::Anonymous, Some("vendor_id"))), + )); + let transport = CannedTransport::new(ok_reply); + + // Act + let results = run( + &coordinator, + &transport, + &snapshot_holding(record_until("p-2", 6000)), + &granting(), + ); + + // Assert + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::NotDue), + "should keep the refreshed UID over the older cookie" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn a_newer_cookie_replaces_an_older_stored_uid() { + let coordinator = coordinator_with(RegisteredProvider::new( + provider_config(""), + RefreshingCookieProvider(cookie_provider(SourceInput::Anonymous, Some("vendor_id"))), + )); + let transport = CannedTransport::new(ok_reply); + + let results = run( + &coordinator, + &transport, + &snapshot_holding(record_until("p-2", 4000)), + &granting(), + ); + + assert!( + matches!(only_status(&results), LookupStatus::Found { kept: 1, .. }), + "should take the cookie UID that outlives the stored one" + ); + } + #[test] fn a_cookie_changing_only_the_agent_type_is_rewritten() { let coordinator = cookie_coordinator(); diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs index 930014d7d..0eec08955 100644 --- a/crates/trusted-server-core/src/ec/provider.rs +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -103,6 +103,19 @@ pub trait IdentityProvider: Send + Sync { let _ = source; false } + + /// Whether a stored UID that outlives the one a cookie carries is kept + /// over the cookie's. + /// + /// For a provider that refreshes its IDs server side, a refresh updates + /// the row but not the browser's cookie, so the next navigation's older + /// cookie would otherwise roll the refreshed UID back. When this returns + /// `true`, a cookie UID whose absolute expiry is earlier than a live + /// stored UID's is ignored; a cookie opt-out still wins. The default + /// always takes the cookie. + fn keeps_fresher_stored_uids(&self) -> bool { + false + } } impl IdentityProvider for Box

{ @@ -138,6 +151,10 @@ impl IdentityProvider for Box

{ fn excluded_where_gdpr_applies(&self, source: &SourceDomain) -> bool { (**self).excluded_where_gdpr_applies(source) } + + fn keeps_fresher_stored_uids(&self) -> bool { + (**self).keeps_fresher_stored_uids() + } } /// What a provider found in the vendor's own first-party cookies. diff --git a/crates/trusted-server-identity-providers/src/uid2.rs b/crates/trusted-server-identity-providers/src/uid2.rs index 43bf3d561..646a53e48 100644 --- a/crates/trusted-server-identity-providers/src/uid2.rs +++ b/crates/trusted-server-identity-providers/src/uid2.rs @@ -287,6 +287,13 @@ impl IdentityProvider for Uid2Provider { true } + /// A server-side refresh does not rewrite the browser's cookies, so a + /// token stored by a refresh outlives the cookie's older token and is + /// kept until the browser SDK refreshes to a newer one. + fn keeps_fresher_stored_uids(&self) -> bool { + true + } + fn build_request( &self, lookup: &LookupInput<'_>, From eab8f7975d9d5ee559bc8475b644997abcd3ea1c Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 17:11:29 +0530 Subject: [PATCH 54/72] Remember only vendor server opt-outs and let a cookie ID clear one Cookie and consent-derived opt-outs no longer get the 30-day lockout: lookups now say whether they remember, keep or forget the provider's opt-out, and only an opt-out a vendor server reports is remembered. The vendor's cookies are read before a remembered opt-out is checked, so an ID in them forgets the opt-out and is stored. ID5's "0" stays an opt-out. Signed-off-by: dhruv8sh --- .../src/ec/after_response.rs | 7 +- .../src/ec/coordinator/mod.rs | 166 +++++++++++++++--- .../src/ec/coordinator/results.rs | 16 ++ crates/trusted-server-core/src/ec/finalize.rs | 3 +- crates/trusted-server-core/src/ec/kv_types.rs | 5 + .../trusted-server-core/src/ec/write_plan.rs | 101 ++++++++++- docs/guide/configuration.md | 9 +- .../2026-10-30-s2s-eid-collection-design.md | 15 +- 8 files changed, 279 insertions(+), 43 deletions(-) diff --git a/crates/trusted-server-core/src/ec/after_response.rs b/crates/trusted-server-core/src/ec/after_response.rs index b46d3ea48..5e5e0f963 100644 --- a/crates/trusted-server-core/src/ec/after_response.rs +++ b/crates/trusted-server-core/src/ec/after_response.rs @@ -134,7 +134,9 @@ pub fn commit_after_response_lookups( #[cfg(test)] mod tests { use super::*; - use crate::ec::coordinator::results::{DropCounts, LookupStatus, ProviderLookup, SkipReason}; + use crate::ec::coordinator::results::{ + DropCounts, LookupStatus, OptOutMemory, ProviderLookup, SkipReason, + }; use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, SourceDomain, UidValue}; use crate::ec::kv::TombstoneOutcome; use crate::ec::kv_types::{KvEid, KvEntry, KvPartnerId, KvUid}; @@ -190,6 +192,7 @@ mod tests { vec![KvEid::new(EidFields::default(), vec![uid])], )], cleared: Vec::new(), + opt_out: OptOutMemory::Unchanged, }], ) } @@ -507,6 +510,7 @@ mod tests { status: LookupStatus::Skipped(SkipReason::NotDue), entries: Vec::new(), cleared: Vec::new(), + opt_out: OptOutMemory::Unchanged, }], ); @@ -578,6 +582,7 @@ mod tests { status: LookupStatus::Skipped(SkipReason::OptedOut), entries: Vec::new(), cleared, + opt_out: OptOutMemory::Unchanged, }], ) } diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index dfbe5d828..2c1afa574 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -15,7 +15,7 @@ use std::time::Duration; use error_stack::Report; -use self::results::{LookupResults, LookupStatus, ProviderLookup, SkipReason}; +use self::results::{LookupResults, LookupStatus, OptOutMemory, ProviderLookup, SkipReason}; use self::transport::{ Deadline, LookupTransport, ProviderIndex, Tagged, TransportError, TransportResponse, }; @@ -391,6 +391,7 @@ impl ProviderCoordinator

{ status, entries, cleared, + opt_out: OptOutMemory::Unchanged, } } @@ -442,11 +443,17 @@ impl ProviderCoordinator

{ return self.lookup(provider, LookupStatus::NoMatch, Vec::new()); } Ok(LookupOutcome::OptedOut) => { - return self.lookup( - provider, - LookupStatus::Skipped(SkipReason::OptedOut), - Vec::new(), - ); + // Only an opt-out the vendor's server reports is remembered: + // it spares the vendor further calls. Cookie opt-outs are + // re-read on every navigation instead. + return ProviderLookup { + opt_out: OptOutMemory::Remember, + ..self.lookup( + provider, + LookupStatus::Skipped(SkipReason::OptedOut), + Vec::new(), + ) + }; } Err(report) => { log::debug!( @@ -548,27 +555,9 @@ impl IdentityCoordinator for ProviderCoordinator

{ value: excluded.clone(), }); } - // A remembered opt-out keeps the provider from being called until - // it lapses. It clears only sources the row still stores, so once - // they are gone the after-response path skips its re-read. - if let Some(entry) = entry.filter(|entry| entry.opted_out(®istered.id, lookup.now)) { - pending.generated.push(Tagged { - provider, - value: ProviderLookup { - provider: registered.id.clone(), - status: LookupStatus::Skipped(SkipReason::OptedOut), - entries: Vec::new(), - cleared: registered - .policy - .sources - .iter() - .filter(|source| entry.ids.contains_key(source.as_str())) - .cloned() - .collect(), - }, - }); - continue; - } + let remembered_opt_out = + entry.filter(|entry| entry.opted_out(®istered.id, lookup.now)); + let mut opted_back_in = false; let (mut due, input_blocked) = Self::due_sources(registered, entry, lookup.now); due.retain(|source| !excluded.contains(source)); // The vendor's own first-party cookies are read on every @@ -597,6 +586,13 @@ impl IdentityCoordinator for ProviderCoordinator

{ recount_found(&mut found); } if matches!(found.status, LookupStatus::Found { .. }) { + // An ID in the vendor's cookies means the visitor + // opted back in, so a remembered opt-out is + // forgotten. + if remembered_opt_out.is_some() { + opted_back_in = true; + found.opt_out = OptOutMemory::Forget; + } // Only the sources the cookies filled are spared a // lookup; the provider's other due sources are // still looked up. @@ -622,7 +618,7 @@ impl IdentityCoordinator for ProviderCoordinator

{ !holds_same_uids_for(entry, source, eids, lookup.now) }); recount_found(&mut found); - if !found.entries.is_empty() { + if !found.entries.is_empty() || opted_back_in { pending.from_cookies.push(Tagged { provider, value: found, @@ -635,6 +631,29 @@ impl IdentityCoordinator for ProviderCoordinator

{ } CookieRead::Absent => {} } + // A remembered opt-out keeps the provider from being called until + // it lapses, unless its cookies hold an ID again. It clears only + // sources the row still stores, so once they are gone the + // after-response path skips its re-read. + if let Some(entry) = remembered_opt_out.filter(|_| !opted_back_in) { + pending.generated.push(Tagged { + provider, + value: ProviderLookup { + provider: registered.id.clone(), + status: LookupStatus::Skipped(SkipReason::OptedOut), + entries: Vec::new(), + cleared: registered + .policy + .sources + .iter() + .filter(|source| entry.ids.contains_key(source.as_str())) + .cloned() + .collect(), + opt_out: OptOutMemory::Unchanged, + }, + }); + continue; + } if due.is_empty() { pending.early.push(skip(if input_blocked { SkipReason::Input @@ -805,6 +824,21 @@ fn exclude_sources(lookup: &mut ProviderLookup, sources: Vec) { /// An opt-out wins; otherwise the found entries of both are kept, and a /// later result that found nothing leaves the cookie result. fn merge_lookups(earlier: ProviderLookup, later: ProviderLookup) -> ProviderLookup { + // A server opt-out is remembered over anything; otherwise the cookies' + // opt-back-in is kept. + let opt_out = match (earlier.opt_out, later.opt_out) { + (_, OptOutMemory::Remember) => OptOutMemory::Remember, + (OptOutMemory::Forget, _) => OptOutMemory::Forget, + (_, memory) => memory, + }; + ProviderLookup { + opt_out, + ..merge_entries(earlier, later) + } +} + +/// Merges the statuses and entries of [`merge_lookups`]. +fn merge_entries(earlier: ProviderLookup, later: ProviderLookup) -> ProviderLookup { let LookupStatus::Found { dropped, .. } = later.status else { return if later.status == LookupStatus::Skipped(SkipReason::OptedOut) || !matches!(earlier.status, LookupStatus::Found { .. }) @@ -2040,6 +2074,7 @@ mod tests { vec![KvEid::new(EidFields::default(), vec![uid])], )], cleared: Vec::new(), + opt_out: OptOutMemory::Unchanged, } } @@ -2049,6 +2084,7 @@ mod tests { status, entries: Vec::new(), cleared: Vec::new(), + opt_out: OptOutMemory::Unchanged, } } @@ -2111,6 +2147,80 @@ mod tests { assert!(transport.sent.borrow().is_empty(), "should send nothing"); } + #[test] + fn a_cookie_id_clears_a_remembered_opt_out() { + // Arrange: the opt-out is remembered for 29 more days, and the + // vendor's cookie holds an ID again. + let coordinator = cookie_coordinator(); + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry.opt_outs.insert( + ProviderId::new("example-provider"), + 1000 + 29 * 24 * 60 * 60, + ); + let transport = CannedTransport::new(ok_reply); + + // Act + let results = run_with_cookies( + &coordinator, + &transport, + &seeded_snapshot(&entry), + &granting(), + &Limiter(false), + "vendor_id=c-1", + ); + + // Assert + let lookup = results.iter().next().expect("should report the provider"); + assert!( + matches!(lookup.status, LookupStatus::Found { kept: 1, .. }), + "should store the cookie ID despite the remembered opt-out" + ); + assert_eq!( + lookup.opt_out, + OptOutMemory::Forget, + "should forget the remembered opt-out" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn only_a_vendor_server_opt_out_is_remembered() { + let cookie_only = cookie_coordinator(); + let cookie = run_with_cookies( + &cookie_only, + &CannedTransport::new(ok_reply), + &live_row(), + &granting(), + &Limiter(false), + "vendor_id=optout", + ); + let server = run( + &coordinator(""), + &CannedTransport::new(optout_reply), + &live_row(), + &granting(), + ); + + let memory = |results: &LookupResults| { + results + .iter() + .next() + .expect("should report the provider") + .opt_out + }; + assert_eq!( + memory(&cookie), + OptOutMemory::Unchanged, + "should not remember a cookie opt-out" + ); + assert_eq!( + memory(&server), + OptOutMemory::Remember, + "should remember a vendor server's opt-out" + ); + } + #[test] fn remembered_opt_out_skips_the_provider_until_it_lapses() { // Arrange diff --git a/crates/trusted-server-core/src/ec/coordinator/results.rs b/crates/trusted-server-core/src/ec/coordinator/results.rs index 6a3144bfe..43a86103f 100644 --- a/crates/trusted-server-core/src/ec/coordinator/results.rs +++ b/crates/trusted-server-core/src/ec/coordinator/results.rs @@ -121,6 +121,20 @@ impl LookupStatus { } } +/// What a lookup does to the provider's remembered opt-out. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum OptOutMemory { + /// Leaves it as it is. + #[default] + Unchanged, + /// Remembers an opt-out the vendor's server reported, so the vendor is + /// not called again for the memory period. + Remember, + /// Forgets it: the vendor's cookies hold an ID again, so the visitor + /// opted back in. + Forget, +} + /// One provider's outcome and validated entries. #[derive(Debug, Clone)] pub struct ProviderLookup { @@ -133,6 +147,8 @@ pub struct ProviderLookup { /// Sources whose stored records are removed, because the user opted /// out of the provider; empty unless `Skipped(OptedOut)`. pub cleared: Vec, + /// What this lookup does to the provider's remembered opt-out. + pub opt_out: OptOutMemory, } /// Validated results of one navigation's lookups. diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index b38541490..5c99b284d 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -566,7 +566,7 @@ mod tests { use crate::consent::types::{ConsentContext, ConsentSource}; use crate::ec::EidSyncSource; use crate::ec::coordinator::results::{ - DropCounts, LookupResults, LookupStatus, ProviderLookup, + DropCounts, LookupResults, LookupStatus, OptOutMemory, ProviderLookup, }; use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, SourceDomain, UidValue}; use crate::ec::kv_types::{KvEid, KvUid}; @@ -2905,6 +2905,7 @@ mod tests { vec![KvEid::new(EidFields::default(), vec![uid])], )], cleared: Vec::new(), + opt_out: OptOutMemory::Unchanged, }], )); let mut response = empty_response(); diff --git a/crates/trusted-server-core/src/ec/kv_types.rs b/crates/trusted-server-core/src/ec/kv_types.rs index 3772f07f3..3a7409190 100644 --- a/crates/trusted-server-core/src/ec/kv_types.rs +++ b/crates/trusted-server-core/src/ec/kv_types.rs @@ -706,6 +706,11 @@ impl KvEntry { ); } + /// Forgets the opt-out remembered for `provider`, if any. + pub fn forget_opt_out(&mut self, provider: &ProviderId) { + self.opt_outs.remove(provider); + } + /// Forgets opt-outs whose time has passed. pub fn prune_opt_outs(&mut self, now: UnixSeconds) { self.opt_outs.retain(|_, until| *until > now.get()); diff --git a/crates/trusted-server-core/src/ec/write_plan.rs b/crates/trusted-server-core/src/ec/write_plan.rs index 23718cd32..c30e0e1cc 100644 --- a/crates/trusted-server-core/src/ec/write_plan.rs +++ b/crates/trusted-server-core/src/ec/write_plan.rs @@ -11,7 +11,7 @@ use std::collections::btree_map; use super::EcKvSnapshot; -use super::coordinator::results::{LookupResults, LookupStatus, SkipReason}; +use super::coordinator::results::{LookupResults, LookupStatus, OptOutMemory, SkipReason}; use super::eid_types::{SourceDomain, UnixSeconds}; use super::kv::PartnerIdUpdate; use super::kv_types::{KvEntry, KvPartnerId, MAX_ENTRY_BYTES}; @@ -130,9 +130,11 @@ fn apply_lookups( let mut written = Vec::new(); for lookup in lookups.iter() { // A newer request may have recorded this provider's opt-out after - // the lookup was dispatched; never restore its UIDs over it. + // the lookup was dispatched; never restore its UIDs over it, unless + // the vendor's cookies showed the visitor opted back in. if entry.opted_out(&lookup.provider, now) && lookup.status != LookupStatus::Skipped(SkipReason::OptedOut) + && lookup.opt_out != OptOutMemory::Forget { log::info!( "EC write planner: discarding results of opted-out provider '{}'", @@ -140,13 +142,16 @@ fn apply_lookups( ); continue; } + if lookup.opt_out == OptOutMemory::Forget { + entry.forget_opt_out(&lookup.provider); + } for (source, eids) in &lookup.entries { if let Some(record) = KvPartnerId::from_eids(eids.clone(), now) { entry.ids.insert(source.as_str().to_owned(), record); written.push(source.clone()); } } - if lookup.status == LookupStatus::Skipped(SkipReason::OptedOut) { + if lookup.opt_out == OptOutMemory::Remember { entry.remember_opt_out(&lookup.provider, now); } for source in &lookup.cleared { @@ -386,6 +391,7 @@ mod tests { vec![KvEid::new(EidFields::default(), uids)], )], cleared: Vec::new(), + opt_out: OptOutMemory::Unchanged, }], ) } @@ -609,6 +615,10 @@ mod tests { } fn failed_lookup(status: LookupStatus) -> LookupResults { + lookup_with_memory(status, OptOutMemory::Unchanged) + } + + fn lookup_with_memory(status: LookupStatus, opt_out: OptOutMemory) -> LookupResults { LookupResults::new( EC_ID, vec![ProviderLookup { @@ -616,10 +626,88 @@ mod tests { status, entries: Vec::new(), cleared: Vec::new(), + opt_out, }], ) } + #[test] + fn a_cookie_opt_out_is_not_remembered() { + // Arrange + let registry = registry(&["ids.example.com"]); + let ingest: [PartnerIdUpdate; 0] = []; + let mut lookups = failed_lookup(LookupStatus::Skipped(SkipReason::OptedOut)); + let cleared = lookups.iter().next().expect("should hold a lookup").clone(); + lookups = LookupResults::new( + EC_ID, + vec![ProviderLookup { + cleared: vec![SourceDomain::try_from("ids.example.com").expect("should parse")], + ..cleared + }], + ); + + // Act + let change = plan_write(&input( + &present(&provider_entry(&["p-1"])), + None, + &ingest, + &lookups, + ®istry, + )) + .expect("should plan update"); + + // Assert + let RowChange::Update { entry, .. } = change else { + panic!("should update existing row") + }; + assert!( + !entry.ids.contains_key("ids.example.com"), + "should clear the opted-out source" + ); + assert!( + entry.opt_outs.is_empty(), + "should not remember an opt-out read from cookies" + ); + } + + #[test] + fn an_opt_back_in_forgets_the_opt_out_and_stores_the_ids() { + // Arrange + let registry = registry(&["ids.example.com"]); + let ingest: [PartnerIdUpdate; 0] = []; + let provider = ProviderId::new("example-provider"); + let mut entry = live_entry(); + entry + .opt_outs + .insert(provider.clone(), 2000 + OPT_OUT_MEMORY_SECONDS); + let found = found("ids.example.com", &["c-1"]); + let lookup = found.iter().next().expect("should hold a lookup").clone(); + let lookups = LookupResults::new( + EC_ID, + vec![ProviderLookup { + opt_out: OptOutMemory::Forget, + ..lookup + }], + ); + + // Act + let change = plan_write(&input(&present(&entry), None, &ingest, &lookups, ®istry)) + .expect("should plan update"); + + // Assert + let RowChange::Update { entry, .. } = change else { + panic!("should update existing row") + }; + assert!( + !entry.opt_outs.contains_key(&provider), + "should forget the remembered opt-out" + ); + assert!( + entry.ids.contains_key("ids.example.com"), + "should store the cookie ID" + ); + } + #[test] fn opt_out_removes_the_provider_sources_and_blocks_browser_fallback() { // Arrange @@ -633,6 +721,7 @@ mod tests { status: LookupStatus::Skipped(SkipReason::OptedOut), entries: Vec::new(), cleared: vec![SourceDomain::try_from("ids.example.com").expect("should parse")], + opt_out: OptOutMemory::Unchanged, }], ); @@ -656,7 +745,10 @@ mod tests { let registry = registry(&["ids.example.com"]); let ingest: [PartnerIdUpdate; 0] = []; let provider = ProviderId::new("example-provider"); - let lookups = failed_lookup(LookupStatus::Skipped(SkipReason::OptedOut)); + let lookups = lookup_with_memory( + LookupStatus::Skipped(SkipReason::OptedOut), + OptOutMemory::Remember, + ); // Act: the first opt-out is remembered. let first = plan_write(&input( @@ -747,6 +839,7 @@ mod tests { status: LookupStatus::Skipped(SkipReason::NotDue), entries: Vec::new(), cleared: Vec::new(), + opt_out: OptOutMemory::Unchanged, }], ); diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 41e56237c..65f214f6d 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -936,9 +936,12 @@ them, and batch sync rejects them and pull sync skips them. Browser-resolved IDs (`ts-eids`, `sharedId`) are never stored for an owned source, so a source a provider cleared on an opt-out is not refilled from the browser. -When a provider reports an opt-out, its sources are cleared and the row -remembers the opt-out for 30 days. The provider is not called for that EC -again until the 30 days pass. +When a provider reports an opt-out, its sources are cleared. An opt-out the +vendor's server reports is also remembered in the row for 30 days, and the +provider is not called for that EC again until the 30 days pass, unless the +vendor's cookies hold an ID again: that means the visitor opted back in, so the +opt-out is forgotten. An opt-out read from the vendor's cookies is not +remembered, because the cookies are read again on every navigation. Some kinds keep private refresh state with a UID, such as a vendor refresh token, and refresh it server side once it is due, while the UID is still live. diff --git a/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md b/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md index a955b2b46..05ac80b59 100644 --- a/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md +++ b/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md @@ -1267,12 +1267,15 @@ For each provider with a `Found` outcome and each source in its response: unowned ones (D8). Each written UID records `owner` and `exp`. - Sources the provider claims but did not return stay unchanged. - `NoMatch`, failures, timeouts and skipped providers leave the row unchanged. -- `OptedOut` (a cookie or a refresh reply) clears every source the provider - claims and records the opt-out in the row's `opt_outs` map for 30 days. Until - it lapses the provider is skipped as `skipped_opted_out` and not called, and - a result from a lookup dispatched before the opt-out was recorded is - discarded rather than written over it. Generated and cookie IDs merge like - `Found`. +- `OptedOut` (a cookie or a vendor server's reply) clears every source the + provider claims. Only an opt-out a vendor server reports is recorded in the + row's `opt_outs` map for 30 days: a cookie opt-out is re-read on every + navigation, so it needs no memory. Until a recorded opt-out lapses the + provider is skipped as `skipped_opted_out` and not called, and a result from + a lookup dispatched before the opt-out was recorded is discarded rather than + written over it. The vendor's cookies are read before the recorded opt-out + is checked: an ID in them means the visitor opted back in, so the opt-out is + forgotten and the ID stored. Generated and cookie IDs merge like `Found`. Browser ingest (§8.4) is applied in the same in-memory merge, before provider results, so the write order never lets browser data overwrite provider data. From 89e0e17289c4d5e6b632865802cb163db1837090 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 17:12:38 +0530 Subject: [PATCH 55/72] Bound provider relative expiry to 30 days in validation Validation already clamped absolute expiries; ProviderExpiry::After is now clamped to the same 30 days, so the bound no longer depends on each provider. Signed-off-by: dhruv8sh --- .../src/ec/coordinator/validate.rs | 35 +++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/validate.rs b/crates/trusted-server-core/src/ec/coordinator/validate.rs index f0a5be90c..bd63245b5 100644 --- a/crates/trusted-server-core/src/ec/coordinator/validate.rs +++ b/crates/trusted-server-core/src/ec/coordinator/validate.rs @@ -13,9 +13,9 @@ use crate::ec::registry::PartnerRegistry; /// Largest private refresh state a provider may keep with one UID. const MAX_REFRESH_STATE_BYTES: usize = 2048; -/// Longest a provider-stated absolute expiry may lie after the request: -/// 30 days. Cookie- and vendor-supplied timestamps are clamped to it. -const MAX_ABSOLUTE_EXPIRY_SECS: u64 = 30 * 24 * 60 * 60; +/// Longest a provider-stated expiry, absolute or relative, may lie after the +/// request: 30 days. Cookie- and vendor-supplied times are clamped to it. +const MAX_EXPIRY_SECS: u64 = 30 * 24 * 60 * 60; /// Policy one response is validated against. pub(crate) struct ValidationPolicy<'a> { @@ -169,8 +169,10 @@ fn stored_uid( fn resolve_expiry(expiry: ProviderExpiry, now: UnixSeconds) -> Option { match expiry { ProviderExpiry::Unspecified => None, - ProviderExpiry::At(at) => Some(at.min(now.saturating_add_secs(MAX_ABSOLUTE_EXPIRY_SECS))), - ProviderExpiry::After(after) => Some(now.saturating_add_secs(after.as_secs())), + ProviderExpiry::At(at) => Some(at.min(now.saturating_add_secs(MAX_EXPIRY_SECS))), + ProviderExpiry::After(after) => { + Some(now.saturating_add_secs(after.as_secs().min(MAX_EXPIRY_SECS))) + } } } @@ -321,6 +323,27 @@ mod tests { } } + #[test] + fn clamps_relative_expiry_to_thirty_days() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + let mut far = uid("u-far"); + far.expiry = ProviderExpiry::After(Duration::from_secs(u64::MAX)); + + let (entries, dropped) = validate( + vec![eid("ids.example.com", vec![far])], + &policy(&provider, &claimed, &partners), + ); + + assert_eq!(dropped, DropCounts::default(), "should drop nothing"); + assert_eq!( + entries[0].1[0].uids()[0].expiry(), + Expiry::At(UnixSeconds::new(1000 + MAX_EXPIRY_SECS)), + "should clamp a far relative expiry to now plus 30 days" + ); + } + #[test] fn clamps_absolute_expiry_to_thirty_days() { let partners = registry(); @@ -340,7 +363,7 @@ mod tests { let uids = entries[0].1[0].uids(); assert_eq!( uids[0].expiry(), - Expiry::At(UnixSeconds::new(1000 + MAX_ABSOLUTE_EXPIRY_SECS)), + Expiry::At(UnixSeconds::new(1000 + MAX_EXPIRY_SECS)), "should clamp a far expiry to now plus 30 days" ); assert_eq!( From 16697b89fbe19bb52568e41df142f0a3284f5bf5 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 17:13:36 +0530 Subject: [PATCH 56/72] Renew an expired generated SharedID instead of replacing it Generation reuses the generated UID the row still holds after it expires, so the ID stays stable for as long as the row lives rather than changing every 30 days. Signed-off-by: dhruv8sh --- .../src/sharedid.rs | 68 +++++++++++++++++-- .../2026-11-02-identity-provider-sharedid.md | 2 +- 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/crates/trusted-server-identity-providers/src/sharedid.rs b/crates/trusted-server-identity-providers/src/sharedid.rs index 62f9dc21d..d6a745bd9 100644 --- a/crates/trusted-server-identity-providers/src/sharedid.rs +++ b/crates/trusted-server-identity-providers/src/sharedid.rs @@ -12,6 +12,8 @@ //! stored, Trusted Server generates one itself: a random UUID v4, stored in //! the row only. No cookie is set, so Prebid's own cookie is never touched; //! when Prebid later writes one, the cookie's ID replaces the generated one. +//! An expired generated ID the row still holds is renewed with the same +//! value, so it stays stable for as long as the row lives. //! Generation is off by default: Prebid may keep its opt-out in localStorage, //! which the server cannot see, so only a publisher who knows its opt-out //! lives in the `_pubcid_optout` cookie should turn it on. @@ -128,13 +130,23 @@ impl IdentityProvider for SharedIdProvider { )]) } - fn generate(&self, _lookup: &LookupInput<'_>) -> Option>> { + fn generate(&self, lookup: &LookupInput<'_>) -> Option>> { if !self.generate { return None; } + // An expired ID the row still holds is renewed rather than replaced, + // so the generated ID stays stable for as long as the row lives. + let id = lookup + .current + .iter() + .filter(|(source, _)| *source == self.source.as_str()) + .flat_map(|(_, eid)| eid.uids()) + .map(|uid| uid.id().as_str().to_owned()) + .next() + .unwrap_or_else(|| Uuid::new_v4().to_string()); Some(vec![device_eid( &self.source, - Cow::Owned(Uuid::new_v4().to_string()), + Cow::Owned(id), // A generated ID has no cookie to renew it, so it keeps the // longest lifetime rather than changing every day. bounded_expiry(Some(MAX_TTL)), @@ -145,7 +157,11 @@ impl IdentityProvider for SharedIdProvider { #[cfg(test)] mod tests { use serde_json::json; - use trusted_server_core::ec::provider::{ProviderExpiry, build_kind}; + use trusted_server_core::ec::eid_types::{ + AgentType, EidFields, Expiry, ProviderId, UidValue, UnixSeconds, + }; + use trusted_server_core::ec::kv_types::{KvEid, KvEntry, KvPartnerId, KvUid}; + use trusted_server_core::ec::provider::{OwnedEntries, ProviderExpiry, build_kind}; use trusted_server_core::settings::{IdentityProviderConfig, ProviderSecrets}; use super::*; @@ -209,6 +225,13 @@ mod tests { } fn generate_with(provider: &dyn IdentityProvider) -> Option>> { + generate_holding(provider, OwnedEntries::empty()) + } + + fn generate_holding( + provider: &dyn IdentityProvider, + current: OwnedEntries<'_>, + ) -> Option>> { let consent = trusted_server_core::consent::ConsentContext::default(); let lookup = LookupInput { ec_id: "ec-example", @@ -219,7 +242,7 @@ mod tests { }, publisher_domain: "www.example.com", due_sources: &[], - current: trusted_server_core::ec::provider::OwnedEntries::empty(), + current, cookies: RequestCookies::default(), hashed_email: None, now: trusted_server_core::ec::eid_types::UnixSeconds::new(1_700_000_000), @@ -235,6 +258,43 @@ mod tests { ); } + #[test] + fn renews_an_expired_generated_id_the_row_still_holds() { + // Arrange: the row holds the generated ID, expired at 1000. + let provider = provider_with(&json!({ "generate": true })); + let uid = KvUid::new( + UidValue::try_from("11111111-2222-4333-8444-555555555555").expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("sharedid")), + Expiry::At(UnixSeconds::new(1000)), + ); + let mut entry = KvEntry::tombstone(1); + entry.ids.insert( + "sharedid.example.com".to_owned(), + KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![uid])], + UnixSeconds::new(500), + ) + .expect("should build record"), + ); + let sources = [SourceDomain::try_from("sharedid.example.com").expect("should parse")]; + let owner = ProviderId::new("sharedid"); + + // Act + let eids = generate_holding( + &*provider, + OwnedEntries::new(Some(&entry), &sources, &owner), + ) + .expect("should generate an ID"); + + // Assert + assert_eq!( + eids[0].uids[0].id, "11111111-2222-4333-8444-555555555555", + "should keep the held ID rather than mint a new one" + ); + } + #[test] fn generates_a_random_uuid_when_none_is_held() { let provider = provider_with(&json!({ "generate": true })); diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-sharedid.md b/docs/superpowers/plans/2026-11-02-identity-provider-sharedid.md index afd79ecb7..cd5544090 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-sharedid.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-sharedid.md @@ -23,7 +23,7 @@ No Lockr capability. - One EID for the single configured source, one UID with `atype = 1`, as Prebid sends `pubcid.org`. - Cookie IDs: default bounded expiry; a cookie repeating the held ID is not rewritten, so the expiry renews only when the source is due. -- Generated IDs expire after 30 days (`MAX_TTL`), not the default 24 hours. +- Generated IDs expire after 30 days (`MAX_TTL`), not the default 24 hours. When one expires while the row still holds it, it is renewed with the same ID rather than replaced by a new UUID, so it stays stable for as long as the row lives. ## Tasks From 579932d5ff8b50dab9198856ccb6b47633e94f06 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 17:14:32 +0530 Subject: [PATCH 57/72] Test that the Fastly adapter joins every Cookie header for providers Signed-off-by: dhruv8sh --- .../trusted-server-adapter-fastly/src/app.rs | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 6b0fb7915..a37078e3c 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -1647,7 +1647,7 @@ mod tests { use trusted_server_core::ec::provider::{ EidLevelFields, EidShape, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderEid, ProviderError, ProviderExpiry, ProviderRequest, - ProviderResponse, ProviderUid, SourceCapability, SourceInput, + ProviderResponse, ProviderUid, RequestCookies, SourceCapability, SourceInput, }; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::TrustedServerError; @@ -2830,6 +2830,29 @@ mod tests { request } + #[test] + fn provider_cookies_join_every_cookie_header() { + // Arrange: HTTP/2 clients may split cookies across headers. + let mut request = navigation_request("/article"); + let headers = request.headers_mut(); + headers.append(header::COOKIE, HeaderValue::from_static("a=1; b=2")); + headers.append(header::COOKIE, HeaderValue::from_static("vendor_id=c-1")); + + // Act + let joined = super::joined_cookie_header(&request); + + // Assert + assert_eq!( + joined, "a=1; b=2; vendor_id=c-1", + "should join every Cookie header" + ); + assert_eq!( + RequestCookies::new(&joined).get("vendor_id"), + Some("c-1"), + "should let providers read a cookie from a later header" + ); + } + fn take_finalize_state(response: &mut Response) -> super::EcFinalizeState { response .extensions_mut() From 54a755f6221d3195daf3cabcc5cf496fe7f47fea Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 17:16:14 +0530 Subject: [PATCH 58/72] Check the held-cookie regressions through the write plan The unchanged-cookie and failed-refresh regressions now run dispatch, collect and plan_write for a 404, timeout, 502, invalid reply and rate limit, each with a no-cookie control, and assert that no write is planned. Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 137 ++++++++++++------ 1 file changed, 93 insertions(+), 44 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 2c1afa574..9e6636f48 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -2692,7 +2692,7 @@ mod tests { }, ]); let config: IdentityProviderConfig = toml::from_str(&format!( - "id = \"example-provider\"\nkind = \"echo\"\nendpoint = \"https://ids.example.com/lookup\"\nallowed_domains = [\"ids.example.com\"]\nsources = [\"ids.example.com\", \"{SECOND_SOURCE}\"]\n" + "id = \"example-provider\"\nkind = \"echo\"\nendpoint = \"https://ids.example.com/lookup\"\nallowed_domains = [\"ids.example.com\"]\nsources = [\"ids.example.com\", \"{SECOND_SOURCE}\"]\nrate_limit_per_hour = 1\n" )) .expect("should parse config"); let partners: Vec = ["ids.example.com", SECOND_SOURCE] @@ -2745,59 +2745,108 @@ mod tests { ); } + /// Plans the row change a navigation's results make, as finalize does. + fn planned_change( + snapshot: &EcKvSnapshot, + results: &LookupResults, + ) -> Option { + let partners: Vec = ["ids.example.com", SECOND_SOURCE] + .into_iter() + .map(|source| { + serde_json::from_value(serde_json::json!({ + "name": source, "source_domain": source, "bidstream_enabled": true + })) + .expect("should build partner") + }) + .collect(); + let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); + crate::ec::write_plan::plan_write(&crate::ec::write_plan::WritePlanInput { + ec_id: EC_ID, + snapshot, + new_entry: None, + browser_ingest: &[], + lookups: results, + registry: ®istry, + now: UnixSeconds::new(1000), + }) + } + + /// Outcomes of a lookup that stores nothing, with a label for messages. + fn failed_outcomes() -> [(&'static str, Reply, bool); 5] { + [ + ("404", not_found_reply, false), + ("timeout", timeout_reply, false), + ("502", server_error_reply, false), + ("invalid", invalid_reply, false), + ("rate-limited", ok_reply, true), + ] + } + #[test] fn an_unchanged_cookie_is_not_rewritten_while_another_source_is_due() { + // Arrange: the row holds the cookie's UID; the second source is due. let coordinator = two_source_coordinator(|provider| provider); - let replies: [Reply; 4] = [ - not_found_reply, - timeout_reply, - server_error_reply, - invalid_reply, - ]; - - for reply in replies { - let transport = CannedTransport::new(reply); - - let results = run(&coordinator, &transport, &held_snapshot("c-1"), &granting()); + let snapshot = held_snapshot("c-1"); + + for (label, reply, limited) in failed_outcomes() { + for cookies in ["vendor_id=c-1", ""] { + let transport = CannedTransport::new(reply); + + // Act + let results = run_with_cookies( + &coordinator, + &transport, + &snapshot, + &granting(), + &Limiter(limited), + cookies, + ); - assert_eq!( - transport.sent.borrow().len(), - 1, - "should still look up the second source" - ); - let lookup = results.iter().next().expect("should report the provider"); - assert!( - lookup.entries.is_empty(), - "should not rewrite the cookie UID the row already holds: {:?}", - lookup.status - ); + // Assert + assert!( + planned_change(&snapshot, &results).is_none(), + "should plan no write for {label} with cookies `{cookies}`: {:?}", + only_status(&results) + ); + } } } #[test] fn a_failed_refresh_keeps_the_held_refresh_state() { - let coordinator = cookie_coordinator(); - let replies: [Reply; 3] = [timeout_reply, server_error_reply, invalid_reply]; - - for reply in replies { - let transport = CannedTransport::new(reply); - - let results = run_with_cookies( - &coordinator, - &transport, - &snapshot_holding(refreshable_record(900)), - &granting(), - &Limiter(false), - "vendor_id=p-1", - ); + // Arrange: the held UID's refresh is due and the cookie repeats it. + let coordinator = coordinator_with(RegisteredProvider::new( + provider_config("rate_limit_per_hour = 1"), + cookie_provider(SourceInput::Anonymous, Some("vendor_id")), + )); + let snapshot = snapshot_holding(refreshable_record(900)); + + for (label, reply, limited) in failed_outcomes() { + for cookies in ["vendor_id=p-1", ""] { + let transport = CannedTransport::new(reply); + + // Act + let results = run_with_cookies( + &coordinator, + &transport, + &snapshot, + &granting(), + &Limiter(limited), + cookies, + ); - assert_eq!(transport.sent.borrow().len(), 1, "should send the refresh"); - let lookup = results.iter().next().expect("should report the provider"); - assert!( - lookup.entries.is_empty(), - "should not replace the held record after a failed refresh: {:?}", - lookup.status - ); + // Assert + let lookup = results.iter().next().expect("should report the provider"); + assert!( + lookup.entries.is_empty(), + "should not replace the held record after {label}: {:?}", + lookup.status + ); + assert!( + planned_change(&snapshot, &results).is_none(), + "should plan no write that drops the refresh state for {label} with cookies `{cookies}`" + ); + } } } From fc231452b86c04467bbf959d140129f21a49d22a Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 17:17:35 +0530 Subject: [PATCH 59/72] Send ID5 the provider, domain and origin fields of its server-side client The fetch body now carries provider (provider_name, defaulting to the publisher domain), domain and origin like ID5's Prebid Server module, instead of o, v, tml and nbPage. Signed-off-by: dhruv8sh --- .../src/id5.rs | 55 ++++++++++++++----- .../plans/2026-11-02-identity-provider-id5.md | 3 +- trusted-server.example.toml | 3 + 3 files changed, 44 insertions(+), 17 deletions(-) diff --git a/crates/trusted-server-identity-providers/src/id5.rs b/crates/trusted-server-identity-providers/src/id5.rs index 04727fe00..2b3eea5b4 100644 --- a/crates/trusted-server-identity-providers/src/id5.rs +++ b/crates/trusted-server-identity-providers/src/id5.rs @@ -27,12 +27,12 @@ //! removed. A fetch reply whose `universal_uid` is `0` reports the same //! opt-out. //! -//! `gpp_sid` is sent comma-joined, as ID5's own server-side client does. -//! The client IP goes in `ip` (IPv4) or `ipv6`. The request field names for -//! the client IP and user agent, and the `o`, `v`, `tml` and `nbPage` -//! fields (ID5's server-side client sends `provider`, `domain` and `origin` -//! instead), follow ID5's server-to-server integration and should be -//! confirmed with ID5 for each partner account. +//! The request follows ID5's server-side client (its Prebid Server module): +//! `gpp_sid` is sent comma-joined, the client IP goes in `ip` (IPv4) or +//! `ipv6`, `provider` names who operates the integration (`provider_name`, +//! default the publisher domain), `domain` is the publisher domain and +//! `origin` is `trusted-server`. The field names for the client IP and user +//! agent should be confirmed with ID5 for each partner account. use std::borrow::Cow; use std::collections::BTreeMap; @@ -70,6 +70,10 @@ fn request_timestamp(now: UnixSeconds) -> Result> .ok_or_else(|| Report::new(ProviderError::Request).attach("current time is out of range")) } +/// Names this integration to ID5 in `origin`, as ID5's Prebid Server module +/// sends `pbs-java`. +const INTEGRATION_ORIGIN: &str = "trusted-server"; + /// The UID ID5 returns for a visitor who opted out. const OPTED_OUT_UID: &str = "0"; @@ -92,6 +96,10 @@ pub struct Id5Settings { /// yielding a usable UID wins. #[serde(default = "Id5Settings::default_cookie_names")] pub cookie_names: Vec, + /// Who operates this integration, sent to ID5 as `provider`. Defaults + /// to the publisher domain. + #[serde(default)] + pub provider_name: Option, /// The Lockr capability; see [`LockrSettings`]. #[serde(default)] pub lockr: LockrSettings, @@ -109,6 +117,7 @@ pub struct Id5Provider { capabilities: ProviderCapabilities, url: Url, partner_id: u32, + provider_name: Option, cookies: CookieReader, } @@ -159,6 +168,7 @@ impl ProviderKind for Id5 { }, url, partner_id: settings.partner_id, + provider_name: settings.provider_name, cookies: CookieReader::with_lockr::( common, settings.cookie_names, @@ -171,10 +181,9 @@ impl ProviderKind for Id5 { #[derive(Serialize)] struct FetchBody<'a> { partner: u32, - v: &'static str, - o: &'static str, - #[serde(rename = "nbPage")] - nb_page: u32, + provider: &'a str, + domain: &'a str, + origin: &'static str, gdpr: u8, #[serde(skip_serializing_if = "Option::is_none")] gdpr_consent: Option<&'a str>, @@ -184,7 +193,6 @@ struct FetchBody<'a> { gpp_string: Option<&'a str>, #[serde(skip_serializing_if = "Option::is_none")] gpp_sid: Option, - tml: &'a str, ts: String, #[serde(skip_serializing_if = "Option::is_none")] ua: Option<&'a str>, @@ -352,15 +360,17 @@ impl IdentityProvider for Id5Provider { let routable_ip = lookup.client.ip.filter(|ip| is_routable_ip(*ip)); let body = FetchBody { partner: self.partner_id, - v: env!("CARGO_PKG_VERSION"), - o: "trusted-server", - nb_page: 1, + provider: self + .provider_name + .as_deref() + .unwrap_or(lookup.publisher_domain), + domain: lookup.publisher_domain, + origin: INTEGRATION_ORIGIN, gdpr: u8::from(consent.gdpr_applies), gdpr_consent: consent.raw_tc_string.as_deref(), us_privacy: consent.raw_us_privacy.as_deref(), gpp_string: consent.raw_gpp_string.as_deref(), gpp_sid: consent.gpp_section_ids.as_deref().map(join_section_ids), - tml: &origin, ts: request_timestamp(lookup.now)?, ua: lookup.client.user_agent, ip: routable_ip.filter(IpAddr::is_ipv4).map(|ip| ip.to_string()), @@ -549,6 +559,21 @@ mod tests { "should send the publisher origin" ); assert_eq!(body["partner"], 1234, "should send the partner"); + assert_eq!( + body["provider"], "www.example.com", + "should name the publisher as operator by default" + ); + assert_eq!( + body["domain"], "www.example.com", + "should send the publisher domain" + ); + assert_eq!( + body["origin"], "trusted-server", + "should name the integration" + ); + for legacy in ["o", "v", "tml", "nbPage"] { + assert!(body.get(legacy).is_none(), "should not send `{legacy}`"); + } let ts = body["ts"].as_str().expect("should send a ts string"); DateTime::parse_from_rfc3339(ts).expect("should send ts as an RFC 3339 timestamp"); assert_eq!(body["gdpr"], 0, "should send that GDPR does not apply"); diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-id5.md b/docs/superpowers/plans/2026-11-02-identity-provider-id5.md index dcf6d7147..e29b1c51a 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-id5.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-id5.md @@ -14,14 +14,13 @@ - No hashed email and no `pd` field: every source is `SourceInput::Anonymous`. - No private provider state, so ID5's `signature` is not sent back. -- No page URL in `LookupInput`, so `tml` is the publisher origin. - Endpoint, sources and the TCF vendor ID come from the common provider config, not hard-coded defaults. - The legacy `universal_uid` fallback goes to the first configured source. An opted-out `universal_uid` of `0` is reported as `LookupOutcome::OptedOut`, detected on the reply as sent (not percent-decoded). ## Request - `POST /.json`, with `Content-Type: application/json` and `Origin: https://`. -- Body: `partner`, `v`, `o = "trusted-server"`, `nbPage = 1`, `gdpr`, and when present `gdpr_consent`, `us_privacy`, `gpp_string`, `gpp_sid` (comma-joined), `ua`, and `ip` (IPv4) or `ipv6` (only when routable, see `common::is_routable_ip`). +- Body, following ID5's server-side client (its Prebid Server module): `partner`, `provider` (`provider_name`, default the publisher domain), `domain` (the publisher domain), `origin = "trusted-server"`, `gdpr`, and when present `gdpr_consent`, `us_privacy`, `gpp_string`, `gpp_sid` (comma-joined), `ua`, and `ip` (IPv4) or `ipv6` (only when routable, see `common::is_routable_ip`). - Confirm the `ip` and `ua` field names with ID5 for each partner account. ## Response diff --git a/trusted-server.example.toml b/trusted-server.example.toml index e37b65404..895e3249d 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -177,6 +177,9 @@ pull_sync_concurrency = 3 # tcf_vendor_id = 9999 # [ec.identity_providers.settings] # partner_id = 1234 +# Optional: who operates this integration, sent to ID5 as `provider`. +# Defaults to the publisher domain. +# provider_name = "publisher.example.com" # cookie_names = ["id5id"] # [ec.identity_providers.settings.lockr] # enabled = true From f68083cbba6094a435b67f6260d4bbaa13c70ec6 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 17:18:39 +0530 Subject: [PATCH 60/72] Skip Lockr cookies where GDPR applies and document the Lockr key overlap Prebid's Lockr AIM module does not run where GDPR applies, so a Lockr cookie on such a request is a leftover; cookie readers now ignore Lockr cookies there. The example tables use Lockr keys that differ from the vendors' own cookies, and the docs say the default Lockr keys are already read as own cookies. Signed-off-by: dhruv8sh --- .../src/ec/coordinator/mod.rs | 15 +++++++++----- crates/trusted-server-core/src/ec/provider.rs | 20 +++++++++++++++++++ .../src/common.rs | 19 ++++++++++++++++-- .../src/id5.rs | 9 +++++++++ ...1-02-identity-provider-lockr-capability.md | 6 +++++- trusted-server.example.toml | 17 +++++++++------- 6 files changed, 71 insertions(+), 15 deletions(-) diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index 9e6636f48..99821f569 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -563,10 +563,12 @@ impl IdentityCoordinator for ProviderCoordinator

{ // The vendor's own first-party cookies are read on every // navigation, so an opt-out or a changed ID is noticed while the // sources still hold live UIDs. Their IDs stand in for a lookup. - match registered - .provider - .read_request_cookies(lookup.cookies.at(lookup.now)) - { + match registered.provider.read_request_cookies( + lookup + .cookies + .at(lookup.now) + .where_gdpr_applies(lookup.consent.gdpr_applies), + ) { CookieRead::OptedOut => { log::debug!( "Identity lookup: provider={} opted out in its cookies", @@ -669,7 +671,10 @@ impl IdentityCoordinator for ProviderCoordinator

{ publisher_domain: lookup.publisher_domain, due_sources: &due, current: OwnedEntries::new(entry, ®istered.policy.sources, ®istered.id), - cookies: lookup.cookies.at(lookup.now), + cookies: lookup + .cookies + .at(lookup.now) + .where_gdpr_applies(lookup.consent.gdpr_applies), now: lookup.now, hashed_email: None, }; diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs index 0eec08955..e942de30b 100644 --- a/crates/trusted-server-core/src/ec/provider.rs +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -195,6 +195,7 @@ impl<'cookie> From>>> for CookieRead<'cookie> { pub struct RequestCookies<'a> { header: &'a str, now: UnixSeconds, + gdpr_applies: bool, } impl Default for RequestCookies<'_> { @@ -224,9 +225,28 @@ impl<'a> RequestCookies<'a> { Self { header, now: UnixSeconds::new(0), + gdpr_applies: false, } } + /// The same cookies, for a request where GDPR does or does not apply. + /// + /// The coordinator sets it from the request consent, for readers whose + /// cookie writers stay off where GDPR applies. + #[must_use] + pub fn where_gdpr_applies(self, gdpr_applies: bool) -> Self { + Self { + gdpr_applies, + ..self + } + } + + /// Whether GDPR applies to the request. + #[must_use] + pub fn gdpr_applies(&self) -> bool { + self.gdpr_applies + } + /// The same cookies, read at request time `now`. /// /// The coordinator sets it before handing the cookies to a provider. diff --git a/crates/trusted-server-identity-providers/src/common.rs b/crates/trusted-server-identity-providers/src/common.rs index 2766500fa..518878aa0 100644 --- a/crates/trusted-server-identity-providers/src/common.rs +++ b/crates/trusted-server-identity-providers/src/common.rs @@ -291,6 +291,21 @@ impl CookieReader { Ok(reader) } + /// The Lockr cookie names to read for this request: none where GDPR + /// applies, because Lockr AIM does not run there, so a Lockr cookie on + /// such a request is a leftover rather than a current delivery. + fn lockr_names<'reader>( + &'reader self, + cookies: RequestCookies<'_>, + ) -> impl Iterator + 'reader { + let names: &[String] = if cookies.gdpr_applies() { + &[] + } else { + &self.lockr + }; + names.iter() + } + /// Whether any cookie name is configured. pub(crate) fn is_empty(&self) -> bool { self.own.is_empty() && self.lockr.is_empty() @@ -301,7 +316,7 @@ impl CookieReader { pub(crate) fn any(&self, cookies: RequestCookies<'_>, matches: impl Fn(&str) -> bool) -> bool { self.own .iter() - .chain(self.lockr.iter()) + .chain(self.lockr_names(cookies)) .flat_map(|name| cookies.get_all(name)) .any(matches) } @@ -322,7 +337,7 @@ impl CookieReader { { return Some(found); } - let (name, found) = self.lockr.iter().find_map(|name| { + let (name, found) = self.lockr_names(cookies).find_map(|name| { let found = cookies.get_all(name).find_map(&mut parse)?; Some((name, found)) })?; diff --git a/crates/trusted-server-identity-providers/src/id5.rs b/crates/trusted-server-identity-providers/src/id5.rs index 2b3eea5b4..f3723427c 100644 --- a/crates/trusted-server-identity-providers/src/id5.rs +++ b/crates/trusted-server-identity-providers/src/id5.rs @@ -959,6 +959,15 @@ mod tests { "should read the Lockr cookie when the own cookie yields nothing" ); assert!(neither.is_none(), "should leave the ID to the lookup"); + assert!( + provider + .read_request_cookies( + RequestCookies::new("example_lockr_id5=lockr-example").where_gdpr_applies(true) + ) + .into_found() + .is_none(), + "should not read Lockr cookies where GDPR applies, as Lockr AIM does not run there" + ); } #[test] diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-lockr-capability.md b/docs/superpowers/plans/2026-11-02-identity-provider-lockr-capability.md index 9254a2c2b..85627a244 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-lockr-capability.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-lockr-capability.md @@ -12,6 +12,10 @@ - `ReadsLockr`: a trait a `ProviderKind` implements to declare the capability, with `LOCKR_COOKIE_NAMES` (`id5id` for ID5, `_lr_env` for LiveRamp, `__uid2_advertising_token` for UID2). `reads_lockr(kind)` in `lib.rs` lists the same kinds. - `CookieReader`: holds the own cookie names and, when Lockr is enabled, the Lockr names not already among them. `read` hands each value to the provider's existing parser: own cookies first, then Lockr cookies. Lookup kinds look up only when neither yields an ID. - A read from a Lockr cookie logs at debug with the provider ID, the cookie name and origin `lockr`. +- Lockr cookies are not read where GDPR applies (`RequestCookies::gdpr_applies`, set by the coordinator from the request consent): Prebid's `lockrAIMIdSystem` stops when `gdprApplies` is true, so a Lockr cookie on such a request is a leftover, not a current delivery. Own cookies are still read. +- Each default Lockr key equals one of the kind's own default cookie names, which are already read, so enabling the capability with the defaults reads nothing extra. Operators list the key configured in Lockr when it differs. + +**Lockr storage format (from Prebid's `lockrAIMIdSystem`):** each vendor token is stored as the raw `advertising_token` under the server-configured `key_name`, in a cookie and or localStorage. Lockr's own `ilui`, `_expiry` and `lockr_expiry_keys` keys are localStorage only. No `inserter` field and no opt-out marker are stored. **Gating:** kinds without the capability (ConnectID, LiveIntent) have `deny_unknown_fields` settings without a `lockr` field, so a `lockr` table fails the provider build at startup. Lockr enabled with an empty `cookie_names` is rejected. @@ -19,7 +23,7 @@ ## Not ported from the reference provider -- The reference captures proxied Lockr API responses: a `revoke-consent` call opts the visitor out of Lockr's sources, and token replies are mapped to sources with Lockr provenance (`inserter`, `mm`) on ID5. The framework has no response-capture hook or opt-out state, and Lockr's default ID5 key is the same `id5id` cookie Prebid writes, so a Lockr-read ID cannot be told apart for provenance. Neither is ported. +- The reference captures proxied Lockr API responses: a `revoke-consent` call opts the visitor out of Lockr's sources, and token replies are mapped to sources with Lockr provenance (`inserter`, `mm`) on ID5. The framework has no response-capture hook or opt-out state, and Lockr's default ID5 key is the same `id5id` cookie Prebid writes, so a Lockr-read ID cannot be told apart for provenance. Neither is ported: Lockr stores neither in browser storage, so a server reading cookies cannot see them. ## Tasks diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 895e3249d..b93a8f38a 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -162,11 +162,14 @@ pull_sync_concurrency = 3 # `storage.type: 'cookie'`; with `html5` storage the lookup runs. # # Lockr capability (id5, liveramp and uid2 only): Lockr AIM writes other -# vendors' IDs into first-party cookies under key names configured per +# vendors' raw tokens into first-party cookies under key names configured per # publisher in Lockr. Enable `[ec.identity_providers.settings.lockr]` to read -# them after the vendor's own cookies and before any lookup; `cookie_names` -# defaults to the vendor's known Lockr key. connectid and liveintent reject a -# `lockr` table. +# them after the vendor's own cookies and before any lookup. Lockr cookies are +# not read where GDPR applies, because Lockr AIM does not run there. The +# default Lockr key of each kind equals one of its own cookie names, which are +# already read, so the capability changes nothing until `cookie_names` lists +# the key configured in Lockr when it differs. connectid and liveintent reject +# a `lockr` table. # [[ec.identity_providers]] # id = "id5" # kind = "id5" @@ -183,7 +186,7 @@ pull_sync_concurrency = 3 # cookie_names = ["id5id"] # [ec.identity_providers.settings.lockr] # enabled = true -# cookie_names = ["id5id"] +# cookie_names = ["example_lockr_id5"] # # LiveIntent (kind "liveintent"): no hashed email, no secrets. Sends the # visitor's first-party LiveIntent cookie (the first present one of @@ -242,7 +245,7 @@ pull_sync_concurrency = 3 # custom_id_namespace = "ana-example" # [ec.identity_providers.settings.lockr] # enabled = true -# cookie_names = ["_lr_env"] +# cookie_names = ["example_lockr_lr_env"] # # Yahoo ConnectID (kind "connectid"): no lookup and no endpoint. Reads # ConnectID from Yahoo's own first-party `connectId` cookie and keeps it until @@ -286,7 +289,7 @@ pull_sync_concurrency = 3 # cookie_names = ["__uid2_advertising_token"] # [ec.identity_providers.settings.lockr] # enabled = true -# cookie_names = ["__uid2_advertising_token"] +# cookie_names = ["example_lockr_uid2"] # # Prebid SharedID (kind "sharedid"): no lookup and no endpoint. Reads the # publisher common ID from Prebid's first-party `_sharedid` cookie, falling From 8f575ad192d6b948223d82a47b54afc7d7e088b9 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 17:19:01 +0530 Subject: [PATCH 61/72] Document the UID2 token-chain mitigations and LiveRamp's Custom ID mapping file Signed-off-by: dhruv8sh --- crates/trusted-server-identity-providers/src/liveramp.rs | 5 ++++- .../plans/2026-11-02-identity-provider-liveramp.md | 2 +- .../plans/2026-11-02-identity-provider-uid2.md | 8 +++++++- trusted-server.example.toml | 5 ++++- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/crates/trusted-server-identity-providers/src/liveramp.rs b/crates/trusted-server-identity-providers/src/liveramp.rs index a40e0b05c..2a9f46f82 100644 --- a/crates/trusted-server-identity-providers/src/liveramp.rs +++ b/crates/trusted-server-identity-providers/src/liveramp.rs @@ -14,7 +14,10 @@ //! stable part of the EC ID, sent as `iv=:` with //! the placement ID, consent strings, the publisher `Origin` and the //! visitor's IP in `X-Forwarded-For`, as a server-to-server call -//! (`atype=3`). `LiveRamp` must enable Custom ID for the placement. A `200` +//! (`atype=3`). `LiveRamp` must enable Custom ID for the placement, and it +//! resolves a Custom ID only through a mapping file the publisher sends it, +//! tying each Custom ID to a hashed email; without one the lookup finds +//! nothing. A `200` //! reply lists `envelopes`; the `RampID` entry (`type` 19) is stored. A //! `204` reply reports an opt-out. Without an `endpoint` no server call is //! made. diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md b/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md index 3b30ea7a8..8708d5c8e 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md @@ -21,7 +21,7 @@ - One EID for the single configured source, one UID with `atype = 3`. - Expiry `ProviderExpiry::After(15 days)` through `bounded_expiry`. -- Optional server lookup (with `endpoint`): `GET /api/identity/v2/envelope` with `pid` (`placement_id`), `it=15`, `iv=:`, `atype=3` (server to server; `2` is the ATS API), consent as `ct`/`cv` (TCF `4`, else US Privacy `3`), `gpp` and `gpp_sid` (the first section ID only); headers `Origin` (publisher) and `X-Forwarded-For` (routable visitor IP, else no lookup). `200 {"envelopes":[{"type":19,"value":…}]}`: the `RampID` entry (`type` 19) is stored like the cookie, and a reply without one is no match; `204` is an opt-out; a blank envelope is no match; other statuses fail. Without `endpoint`, no server call. LiveRamp must enable Custom ID for the placement and approve the origin. +- Optional server lookup (with `endpoint`): `GET /api/identity/v2/envelope` with `pid` (`placement_id`), `it=15`, `iv=:`, `atype=3` (server to server; `2` is the ATS API), consent as `ct`/`cv` (TCF `4`, else US Privacy `3`), `gpp` and `gpp_sid` (the first section ID only); headers `Origin` (publisher) and `X-Forwarded-For` (routable visitor IP, else no lookup). `200 {"envelopes":[{"type":19,"value":…}]}`: the `RampID` entry (`type` 19) is stored like the cookie, and a reply without one is no match; `204` is an opt-out; a blank envelope is no match; other statuses fail. Without `endpoint`, no server call. LiveRamp must enable Custom ID for the placement and approve the origin. LiveRamp resolves a Custom ID only through a mapping file the publisher sends it, which ties each Custom ID to a hashed email; without that file, or for an EC ID absent from it, the lookup finds nothing. ## Lockr diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md b/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md index b53fc27ae..d21a690c2 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md @@ -31,7 +31,13 @@ - Token generation (needs a hashed email; out of scope). -Risk: a server refresh does not rewrite `__uid_2`. Refreshing does not invalidate the previous token or refresh token, so the server refresh and the browser SDK's own refresh both succeed and fork into two token chains. A later navigation whose cookie still carries the older token then replaces the refreshed row token; `rate_limit_per_hour` bounds the resulting refreshes. +Risk: a server refresh does not rewrite `__uid_2`. Refreshing does not invalidate the previous token or refresh token, so the server refresh and the browser SDK's own refresh both succeed and fork into two token chains. Mitigations: + +- The provider sets `keeps_fresher_stored_uids`: a cookie token whose `identity_expires` is earlier than the live stored token's is ignored, so an older cookie never rolls a refreshed token back, and the refreshed token's own `refresh_from` decides the next server refresh. +- Once the SDK refreshes, its cookie token outlives the stored one and replaces it with its refresh state, so the row follows the SDK's chain again; the two chains do not alternate on every navigation. +- A failed refresh keeps the held token and refresh state, and due state is kept even past `refresh_from`; `rate_limit_per_hour` bounds the retries. + +Remaining risk: while the SDK has not refreshed, the server refresh keeps a second chain alive, one refresh per `refresh_from` interval. Operators running the SDK with `useCookie: true` can leave `endpoint` unset to rely on the SDK alone. ## Tasks diff --git a/trusted-server.example.toml b/trusted-server.example.toml index b93a8f38a..81c2cb939 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -230,7 +230,10 @@ pull_sync_concurrency = 3 # cookie is looked up by Custom ID: the stable part of the EC ID, sent as # `:` with the visitor IP and consent. LiveRamp must # enable Custom ID for `placement_id` and approve the publisher origin; both -# settings are required with an endpoint. No login or hashed email is used. +# settings are required with an endpoint. LiveRamp resolves a Custom ID only +# through a mapping file the publisher sends it, tying each ID to a hashed +# email; without one the lookup finds nothing. Trusted Server itself sends no +# login or hashed email. # [[ec.identity_providers]] # id = "liveramp" # kind = "liveramp" From 5e017444fd8dba5917e75955b8127d70c9d1cba8 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 17:30:34 +0530 Subject: [PATCH 62/72] Drop placeholder LiveIntent IDs and skip untyped LiveRamp envelopes LiveIntent now applies the shared usable-ID check, so a blank or 0 ID reads as no match. A LiveRamp envelope entry without a type no longer fails the whole reply. Signed-off-by: dhruv8sh --- .../src/liveintent.rs | 8 ++++++-- .../src/liveramp.rs | 19 ++++++++++++++++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/crates/trusted-server-identity-providers/src/liveintent.rs b/crates/trusted-server-identity-providers/src/liveintent.rs index aeebb8723..f0f6b8d48 100644 --- a/crates/trusted-server-identity-providers/src/liveintent.rs +++ b/crates/trusted-server-identity-providers/src/liveintent.rs @@ -37,7 +37,7 @@ use trusted_server_core::ec::provider::{ }; use url::Url; -use crate::common::{DEVICE_ATYPE, bounded_expiry, is_routable_ip, join_section_ids}; +use crate::common::{DEVICE_ATYPE, bounded_expiry, is_routable_ip, is_usable_id, join_section_ids}; /// Each attribute `LiveIntent` resolves and the source it carries. pub const ATTRIBUTE_SOURCES: &[(&str, &str)] = &[ @@ -362,7 +362,7 @@ impl IdentityProvider for LiveIntentProvider { let raw = reply.get(attribute.as_str())?; let id = Cow::<'body, str>::deserialize(*raw) .ok() - .filter(|id| !id.is_empty())?; + .filter(|id| is_usable_id(id))?; // Prebid sends the Trade Desk ID LiveIntent resolved as a // device-based ID (atype 1), like Unified ID's TDID. let (matcher, eid_ext, uid_ext, atype) = match attribute.as_str() { @@ -807,9 +807,13 @@ mod tests { let headers = HeaderMap::new(); let empty_id = serde_json::to_vec(&json!({ "nonId": "" })).expect("should encode body"); + let blank_id = serde_json::to_vec(&json!({ "nonId": " " })).expect("should encode body"); + let zero_id = serde_json::to_vec(&json!({ "nonId": "0" })).expect("should encode body"); for (status, body) in [ (StatusCode::NO_CONTENT, &b""[..]), (StatusCode::OK, &empty_id[..]), + (StatusCode::OK, &blank_id[..]), + (StatusCode::OK, &zero_id[..]), ] { let outcome = provider .parse_response(ProviderResponse { diff --git a/crates/trusted-server-identity-providers/src/liveramp.rs b/crates/trusted-server-identity-providers/src/liveramp.rs index 2a9f46f82..9b38e4653 100644 --- a/crates/trusted-server-identity-providers/src/liveramp.rs +++ b/crates/trusted-server-identity-providers/src/liveramp.rs @@ -178,8 +178,8 @@ struct EnvelopeReply { /// One entry of [`EnvelopeReply::envelopes`]. #[derive(Deserialize)] struct ReplyEnvelope { - #[serde(rename = "type")] - kind: u32, + #[serde(rename = "type", default)] + kind: Option, #[serde(default)] value: Option, } @@ -300,7 +300,7 @@ impl IdentityProvider for LiveRampProvider { let Some(envelope) = reply .envelopes .into_iter() - .find(|envelope| envelope.kind == RAMPID_ENVELOPE_TYPE) + .find(|envelope| envelope.kind == Some(RAMPID_ENVELOPE_TYPE)) .and_then(|envelope| envelope.value) .filter(|envelope| is_usable_id(envelope)) else { @@ -749,6 +749,19 @@ mod tests { fn maps_no_content_to_an_opt_out_and_a_blank_envelope_to_no_match() { let blank = json!({"envelopes": [{"type": 19, "value": "", "err": null}]}).to_string(); let other = json!({"envelopes": [{"type": 25, "value": "pair-example"}]}).to_string(); + let untyped = json!({"envelopes": [ + {"value": "untyped-example"}, + {"type": 19, "value": "rampid-example"}, + ]}) + .to_string(); + + assert!( + matches!( + reply(StatusCode::OK, untyped.as_bytes()), + Ok(LookupOutcome::Found(eids)) if eids[0].uids[0].id == "rampid-example" + ), + "should skip an untyped entry and keep the RampID envelope" + ); assert!( matches!( From 9b0809a447fab0c72f6f3d4977f8987f35989fc9 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 17:30:44 +0530 Subject: [PATCH 63/72] Log review rounds 7 and 8 for the vendor identity providers Signed-off-by: dhruv8sh --- .../2026-10-30-s2s-eid-collection-design.md | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md b/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md index 05ac80b59..646edb661 100644 --- a/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md +++ b/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md @@ -1785,3 +1785,37 @@ combinations and adapter differences, on the tree after round 5's fix. - Non-blocking, no change: only the Fastly adapter builds the registry with provider ownership and vendor gates (§12.3 "On Fastly"); per-adapter deploy validation stays with #1552. + +### Round 7 (2026-11-12, PR #1488 at 5d22274c1, Claude Code (Opus 5.5) review agent) + +Angle: data flow and edge cases end to end per provider (UID2, ID5, +LiveRamp, ConnectID, LiveIntent, SharedID, Criteo, Lockr): cookie parsing, +request building, reply parsing, opt-out, 204 and 404 handling, refresh, +expiry, and consent and region gating. + +- No blocking findings. +- Non-blocking, fixed in 371e0ccba: LiveIntent kept a blank or `0` ID from + the reply; it now uses the shared usable-ID check. A LiveRamp envelope + entry without `type` failed the whole reply; it is now skipped. +- Non-blocking, no change: Lockr's storage format was checked against + Prebid's `lockrAIMIdSystem`. It writes only the raw `advertising_token` + under `key_name` to a cookie, with no `inserter` and no revoke marker, so + neither can be read from cookies. +- Known, open: LiveIntent partner UIDs carry `ext.provider` of + `liveintent.com` rather than the page domain, and a reply's `m` is not + copied into the `nonId` EID's `ext`. + +### Round 8 (2026-11-12, PR #1488 at 371e0ccba, Claude Code (Opus 5.5) review agent) + +Angle: adversarial. Malformed cookies and replies, config combinations, +clock edges and privacy leaks. + +- No blocking findings. Re-checked non-integer, negative, huge and NaN + expiry and TTL values (rejected, clamped to 30 days in validation, or + read as expired); empty and duplicate cookies, a stale opt-out cookie + beside a live ID (the opt-out wins); Lockr names equal to own names, Lockr + with empty own cookies, and Lockr under GDPR; and that no cookie value or + token is logged. +- Non-blocking, no change: a UID2 refresh that keeps failing is retried on + every navigation until the token expires unless `rate_limit_per_hour` is + set, as round 2 of #1488's review accepted. From 9302cf685cefc50f04d6233d6135b7b21cf8c0a9 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 17:30:49 +0530 Subject: [PATCH 64/72] Name the commit that bounds UID2 refresh retries in the review log Signed-off-by: dhruv8sh --- docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md b/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md index 646edb661..936574a10 100644 --- a/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md +++ b/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md @@ -1818,4 +1818,4 @@ clock edges and privacy leaks. token is logged. - Non-blocking, no change: a UID2 refresh that keeps failing is retried on every navigation until the token expires unless `rate_limit_per_hour` is - set, as round 2 of #1488's review accepted. + set. The fix for overdue UID2 refresh state (834af2591) chose this bound. From b796282ca5db8a37e0c42c122e77f3de3d36ffe1 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 18:14:04 +0530 Subject: [PATCH 65/72] Name the publisher domain as the LiveIntent TDID provider and keep m Prebid's LiveIntent module sets the resolved Trade Desk ID's ext.provider to the page domain, falling back to liveintent.com, and copies the reply's m next to segments in the nonId EID ext. Carry the publisher domain to parse_response through the request context and build both ext values the same way. Signed-off-by: dhruv8sh --- .../src/liveintent.rs | 234 +++++++++++++++--- 1 file changed, 193 insertions(+), 41 deletions(-) diff --git a/crates/trusted-server-identity-providers/src/liveintent.rs b/crates/trusted-server-identity-providers/src/liveintent.rs index f0f6b8d48..86950645d 100644 --- a/crates/trusted-server-identity-providers/src/liveintent.rs +++ b/crates/trusted-server-identity-providers/src/liveintent.rs @@ -8,10 +8,11 @@ //! sources. No hashed email is sent. //! //! As in Prebid's `LiveIntent` module, every attribute other than `nonId` is -//! a partner ID and its UID carries `ext.provider` naming `LiveIntent`; the -//! Trade Desk ID also carries `ext.rtiPartner` and its EID names -//! `LiveIntent` as `matcher`. A non-empty `segments` array in the reply -//! becomes the `nonId` EID's `ext`. +//! a partner ID and its UID carries `ext.provider` naming `LiveIntent`. The +//! Trade Desk ID instead carries `ext.rtiPartner` and an `ext.provider` +//! naming the publisher's domain, and its EID names `LiveIntent` as +//! `matcher`. A non-empty `segments` array and a non-empty `m` value in the +//! reply go in the `nonId` EID's `ext`. //! //! This kind has no Lockr capability: Lockr AIM is not documented to //! deliver `LiveIntent` IDs, so a `lockr` settings table is rejected. @@ -65,12 +66,11 @@ const UID2_SOURCE: &str = "uidapi.com"; /// The attribute holding the Trade Desk ID `LiveIntent` resolved. const TDID_ATTRIBUTE: &str = "thetradedesk"; -/// `matcher` of the Trade Desk ID EID: `LiveIntent` matched it. +/// `matcher` of the Trade Desk ID EID: `LiveIntent` matched it. It is also +/// the Trade Desk ID's `ext.provider` when the publisher domain is unknown, +/// as in Prebid. const TDID_MATCHER: &str = "liveintent.com"; -/// UID `ext` Prebid puts on the Trade Desk ID that `LiveIntent` resolved. -const TDID_UID_EXT: &str = r#"{"rtiPartner":"TDID","provider":"liveintent.com"}"#; - /// `LiveIntent`'s default first-party ID cookie on the publisher's domain. const FIRST_PARTY_COOKIE: &str = "_lc2_fpi"; @@ -134,7 +134,6 @@ pub struct LiveIntentProvider { attributes: Box<[(String, SourceDomain)]>, cookie_names: Box<[String]>, partner_uid_ext: &'static RawValue, - tdid_uid_ext: &'static RawValue, } /// Whether `id` has the `did-` form `LiveIntent` accepts; it answers any @@ -192,8 +191,6 @@ impl ProviderKind for LiveIntent { } let partner_uid_ext = serde_json::from_str::<&'static RawValue>(PARTNER_UID_EXT) .change_context(ProviderConfigError::Invalid)?; - let tdid_uid_ext = serde_json::from_str::<&'static RawValue>(TDID_UID_EXT) - .change_context(ProviderConfigError::Invalid)?; let distributor = settings.distributor_id.as_deref().unwrap_or("prebid"); let mut url = common .endpoint @@ -229,7 +226,6 @@ impl ProviderKind for LiveIntent { attributes, cookie_names: settings.cookie_names.into_boxed_slice(), partner_uid_ext, - tdid_uid_ext, }) } } @@ -316,7 +312,8 @@ impl IdentityProvider for LiveIntentProvider { url, headers, body: Bytes::new(), - context: None, + // The publisher domain names the Trade Desk ID's provider. + context: Some(lookup.publisher_domain.to_owned()), }) } @@ -334,27 +331,8 @@ impl IdentityProvider for LiveIntentProvider { let reply: HashMap = serde_json::from_slice(response.body).change_context(ProviderError::InvalidResponse)?; let expiry = expiry(response.headers); - // Only a non-empty segment array is kept, as Prebid does. - let segments = reply - .get("segments") - .filter(|segments| { - serde_json::from_str::>(segments.get()) - .is_ok_and(|segments| !segments.is_empty()) - }) - .and_then(|segments| { - RawValue::from_string(format!(r#"{{"segments":{}}}"#, segments.get())).ok() - }) - // An oversized EID `ext` would drop the whole `nonId` EID, so the - // segments are left out rather than the ID. - .filter(|ext| { - let fits = ext.get().len() <= MAX_EXT_BYTES; - if !fits { - log::debug!( - "LiveIntent: omitting segments over {MAX_EXT_BYTES} bytes from the nonId EID" - ); - } - fits - }); + let non_id_ext = non_id_ext(&reply); + let tdid_uid_ext = tdid_uid_ext(response.context)?; let eids: Vec> = self .attributes .iter() @@ -368,17 +346,22 @@ impl IdentityProvider for LiveIntentProvider { let (matcher, eid_ext, uid_ext, atype) = match attribute.as_str() { NON_ID_ATTRIBUTE => ( None, - segments.clone().map(Cow::Owned), + non_id_ext.clone().map(Cow::Owned), None, LIVEINTENT_ATYPE, ), TDID_ATTRIBUTE => ( Some(Cow::Borrowed(TDID_MATCHER)), None, - Some(self.tdid_uid_ext), + Some(Cow::Owned(tdid_uid_ext.clone())), DEVICE_ATYPE, ), - _ => (None, None, Some(self.partner_uid_ext), LIVEINTENT_ATYPE), + _ => ( + None, + None, + Some(Cow::Borrowed(self.partner_uid_ext)), + LIVEINTENT_ATYPE, + ), }; Some(ProviderEid { source: Cow::Owned(source.as_str().to_owned()), @@ -389,7 +372,7 @@ impl IdentityProvider for LiveIntentProvider { uids: vec![ProviderUid { id, atype: Some(atype), - ext: uid_ext.map(Cow::Borrowed), + ext: uid_ext, expiry, refresh: None, }], @@ -404,6 +387,65 @@ impl IdentityProvider for LiveIntentProvider { } } +/// Builds the `nonId` EID `ext` from the reply's `segments` and `m`, as +/// Prebid's `LiveIntent` module does. +/// +/// Only a non-empty `segments` array and a truthy `m` are kept. An +/// oversized `ext` would drop the whole `nonId` EID, so the segments are +/// left out first, then `m`, rather than the ID. +fn non_id_ext(reply: &HashMap) -> Option> { + let field = |name: &str| -> Option { + serde_json::from_str(reply.get(name)?.get()).ok() + }; + let segments = field("segments").filter(|segments| { + segments + .as_array() + .is_some_and(|segments| !segments.is_empty()) + }); + let m = field("m").filter(|m| { + !matches!(m, serde_json::Value::Null | serde_json::Value::Bool(false)) + && m.as_str() != Some("") + && m.as_f64() != Some(0.0) + }); + let build = |segments: Option<&serde_json::Value>| -> Option> { + let mut ext = serde_json::Map::new(); + if let Some(segments) = segments { + ext.insert("segments".to_owned(), segments.clone()); + } + if let Some(m) = &m { + ext.insert("m".to_owned(), m.clone()); + } + if ext.is_empty() { + return None; + } + let ext = serde_json::value::to_raw_value(&ext).ok()?; + if ext.get().len() > MAX_EXT_BYTES { + log::debug!("LiveIntent: omitting a nonId EID ext over {MAX_EXT_BYTES} bytes"); + return None; + } + Some(ext) + }; + build(segments.as_ref()).or_else(|| segments.as_ref().and_then(|_| build(None))) +} + +/// Builds the Trade Desk ID's UID `ext`: Prebid names the page's domain as +/// its `provider`, falling back to `LiveIntent`'s domain. +/// +/// # Errors +/// +/// Returns [`ProviderError::InvalidResponse`] if the `ext` cannot be +/// serialized. +fn tdid_uid_ext(publisher_domain: Option<&str>) -> Result, Report> { + let provider = publisher_domain + .filter(|domain| !domain.is_empty()) + .unwrap_or(TDID_MATCHER); + serde_json::value::to_raw_value(&serde_json::json!({ + "rtiPartner": "TDID", + "provider": provider, + })) + .change_context(ProviderError::InvalidResponse) +} + #[cfg(test)] mod tests { use serde_json::json; @@ -703,9 +745,13 @@ mod tests { "should name LiveIntent as the TDID matcher" ); assert_eq!( - eids[1].uids[0].ext.as_deref().map(RawValue::get), - Some(r#"{"rtiPartner":"TDID","provider":"liveintent.com"}"#), - "should mark the TDID as LiveIntent-provided" + eids[1].uids[0] + .ext + .as_deref() + .map(|ext| serde_json::from_str::(ext.get()) + .expect("should parse TDID ext")), + Some(json!({ "rtiPartner": "TDID", "provider": "liveintent.com" })), + "should fall back to LiveIntent as the TDID provider without a publisher domain" ); assert!(eids[1].ext.is_none(), "should keep segments on nonId only"); assert_eq!(eids[2].source, "pubmatic.com", "should map a partner ID"); @@ -716,6 +762,112 @@ mod tests { ); } + #[test] + fn names_the_publisher_domain_as_the_tdid_provider() { + let provider = provider(); + let headers = HeaderMap::new(); + let consent = ConsentContext::default(); + let due = due(); + let lookup = LookupInput { + ec_id: "ec-example", + consent: &consent, + client: ClientSignals { + ip: None, + user_agent: None, + }, + publisher_domain: "www.example.com", + due_sources: &due, + current: OwnedEntries::empty(), + cookies: RequestCookies::default(), + hashed_email: None, + now: trusted_server_core::ec::eid_types::UnixSeconds::new(1_700_000_000), + }; + let request = provider + .build_request(&lookup) + .expect("should build request"); + let body = json!({ "thetradedesk": "tdid-example" }) + .to_string() + .into_bytes(); + + let outcome = provider + .parse_response(ProviderResponse { + status: StatusCode::OK, + headers: &headers, + body: &body, + context: request.context.as_deref(), + }) + .expect("should parse response"); + let LookupOutcome::Found(eids) = outcome else { + panic!("should find the TDID EID"); + }; + + let ext: serde_json::Value = serde_json::from_str( + eids[0].uids[0] + .ext + .as_deref() + .expect("should set the TDID ext") + .get(), + ) + .expect("should parse TDID ext"); + assert_eq!( + ext, + json!({ "rtiPartner": "TDID", "provider": "www.example.com" }), + "should name the publisher domain as the TDID provider, as Prebid does" + ); + assert_eq!( + eids[0].matcher.as_deref(), + Some("liveintent.com"), + "should keep LiveIntent as the TDID matcher" + ); + } + + #[test] + fn copies_m_into_the_non_id_ext() { + let provider = provider(); + let headers = HeaderMap::new(); + let cases = [ + ( + json!({ "nonId": "li-example", "segments": ["s1"], "m": "m-example" }), + Some(json!({ "segments": ["s1"], "m": "m-example" })), + ), + ( + json!({ "nonId": "li-example", "m": "m-example" }), + Some(json!({ "m": "m-example" })), + ), + (json!({ "nonId": "li-example", "m": "" }), None), + (json!({ "nonId": "li-example", "m": null }), None), + ( + json!({ + "nonId": "li-example", + "segments": (0..120).map(|n| format!("seg{n:04}")).collect::>(), + "m": "m-example", + }), + Some(json!({ "m": "m-example" })), + ), + ]; + + for (reply, expected) in cases { + let body = reply.to_string().into_bytes(); + let outcome = provider + .parse_response(ProviderResponse { + status: StatusCode::OK, + headers: &headers, + body: &body, + context: None, + }) + .expect("should parse response"); + let LookupOutcome::Found(eids) = outcome else { + panic!("should find the nonId EID"); + }; + + let ext = eids[0].ext.as_deref().map(|ext| { + serde_json::from_str::(ext.get()) + .expect("should parse nonId ext") + }); + assert_eq!(ext, expected, "should build the nonId ext for {reply}"); + } + } + #[test] fn rejects_a_distributor_id_liveintent_did_not_issue() { for distributor_id in ["prebid", "did-", "did-01/2"] { From c0cfaaf63aa171bb3b65c3064dde4fd4d8315f3c Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 18:29:47 +0530 Subject: [PATCH 66/72] Withhold stored UIDs of GDPR-excluded sources from egress The coordinator drops sources a provider excludes where GDPR applies only when it writes the row, so a UID2 UID stored outside GDPR still reached bidders until the next committed write. Mark those sources in the partner registry for every configured provider and skip them in live_eids for a GDPR visitor. Signed-off-by: dhruv8sh --- .../src/auction/endpoints.rs | 40 +++++++++++ crates/trusted-server-core/src/ec/eids.rs | 39 ++++++++++- .../trusted-server-core/src/ec/pull_sync.rs | 1 + crates/trusted-server-core/src/ec/registry.rs | 30 +++++++++ .../src/lib.rs | 67 ++++++++++++++++++- .../src/liveintent.rs | 8 ++- .../src/uid2.rs | 10 ++- 7 files changed, 189 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 1155a786d..38f43d607 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -2054,6 +2054,46 @@ mod tests { ); } + #[test] + fn resolve_auction_eids_withholds_a_gdpr_excluded_source_where_gdpr_applies() { + // Arrange: the row holds a UID2 UID written while the visitor was + // outside GDPR; the visitor now arrives under GDPR with TCF Purposes + // 1 and 4 granted. + let mut registry = partner_registry(&["uidapi.com", "ids.example.com"]); + assert!( + registry.exclude_where_gdpr_applies("uidapi.com"), + "should mark the UID2 source" + ); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let mut entry = crate::ec::kv_types::KvEntry::minimal("uidapi.com", "uid2-token", 1); + entry.ids.insert( + "ids.example.com".to_owned(), + crate::ec::kv_types::KvPartnerId::single("other-uid"), + ); + entry.consent.ok = true; + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(entry), + generation: None, + }; + let ec_context = EcContext::new_for_test( + Some(ec_id), + crate::test_support::tests::consent_granting_vendors(vec![1]), + ); + + // Act + let eids = resolve_auction_eids(&snapshot, Some(®istry), &ec_context) + .expect("should resolve EIDs under granted consent"); + + // Assert + let sources: Vec<&str> = eids.iter().map(|eid| eid.source.as_str()).collect(); + assert_eq!( + sources, + vec!["ids.example.com"], + "should not send the GDPR-excluded source to bidders" + ); + } + #[test] fn resolve_auction_eids_returns_none_without_registry() { let ec_id = format!("{}.ABC123", "a".repeat(64)); diff --git a/crates/trusted-server-core/src/ec/eids.rs b/crates/trusted-server-core/src/ec/eids.rs index 9e7a4bad6..f1073b244 100644 --- a/crates/trusted-server-core/src/ec/eids.rs +++ b/crates/trusted-server-core/src/ec/eids.rs @@ -105,7 +105,8 @@ impl KvEntry { /// source order, borrowing from the row. /// /// A source whose owning provider has a TCF vendor ID is skipped while - /// `consent` withholds that vendor's consent. + /// `consent` withholds that vendor's consent, and a source a provider + /// excludes where GDPR applies is skipped for a GDPR visitor. pub fn live_eids<'row>( &'row self, registry: &'row PartnerRegistry, @@ -114,7 +115,9 @@ impl KvEntry { ) -> impl Iterator> + 'row { self.ids.iter().flat_map(move |(source, record)| { let partner = registry.get(source).filter(|partner| { - partner.bidstream_enabled && !partner.owner_vendor_consent_denied(consent) + partner.bidstream_enabled + && !partner.owner_vendor_consent_denied(consent) + && !partner.excluded_for(consent) }); partner .into_iter() @@ -330,6 +333,38 @@ mod tests { ); } + #[test] + fn live_eids_withhold_a_gdpr_excluded_source_for_a_gdpr_visitor() { + // Arrange: a stored UID for a source a provider excludes where GDPR + // applies, written while the visitor was outside GDPR. + let mut registry = registry_with("ids.example.com", true); + assert!( + registry.exclude_where_gdpr_applies("ids.example.com"), + "should mark the source" + ); + let mut entry = KvEntry::minimal("ids.example.com", "placeholder", 1); + entry.ids.insert( + "ids.example.com".to_owned(), + provider_record(&[("p-1", Expiry::Never)]), + ); + let gdpr = consent_granting_vendors(vec![1]); + + // Act + let under_gdpr = entry + .live_eids(®istry, &gdpr, UnixSeconds::new(0)) + .count(); + let elsewhere = entry + .live_eids(®istry, &ConsentContext::default(), UnixSeconds::new(0)) + .count(); + + // Assert + assert_eq!( + under_gdpr, 0, + "should withhold the excluded source with TCF Purposes 1 and 4 granted" + ); + assert_eq!(elsewhere, 1, "should serve the source outside GDPR"); + } + #[test] fn live_eids_respect_bidstream_flag() { let registry = registry_with("ids.example.com", false); diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index d5064a984..fc1f89a86 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -562,6 +562,7 @@ mod tests { ts_pull_token: Some(Redacted::new("token".to_owned())), provider_owned: false, owner_tcf_vendor_ids: Vec::new(), + excluded_where_gdpr_applies: false, } } diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index 4e840cd28..fe20f8312 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -9,6 +9,7 @@ use std::num::NonZeroU16; use error_stack::{Report, ResultExt as _}; +use crate::consent::jurisdiction::Jurisdiction; use crate::consent::{ConsentContext, tcf_vendor_consent_denied}; use crate::error::TrustedServerError; use crate::redacted::Redacted; @@ -56,6 +57,10 @@ pub struct PartnerConfig { /// claim this source. Its UIDs leave the server only while every one of /// these vendors has consent. pub owner_tcf_vendor_ids: Vec, + /// Whether a configured provider, enabled or disabled, excludes this + /// source where GDPR applies. Its UIDs then never leave the server for a + /// visitor whose jurisdiction is GDPR. + pub excluded_where_gdpr_applies: bool, } impl PartnerConfig { @@ -67,6 +72,13 @@ impl PartnerConfig { .iter() .any(|vendor| tcf_vendor_consent_denied(consent, *vendor)) } + + /// Whether this source's UIDs are withheld because a provider excludes it + /// and the visitor's jurisdiction is GDPR. + #[must_use] + pub fn excluded_for(&self, consent: &ConsentContext) -> bool { + self.excluded_where_gdpr_applies && matches!(consent.jurisdiction, Jurisdiction::Gdpr) + } } /// In-memory partner registry with O(1) lookups by source domain and API key hash. @@ -337,6 +349,23 @@ impl PartnerRegistry { } } + /// Marks `source` as excluded where GDPR applies, so its stored UIDs are + /// withheld from egress for a visitor whose jurisdiction is GDPR. + /// + /// Returns `false` when the source has no `[[ec.partners]]` entry. + pub fn exclude_where_gdpr_applies(&mut self, source: &str) -> bool { + let Ok(normalized) = normalize_partner_source_domain(source) else { + return false; + }; + match self.by_source_domain.get_mut(&normalized) { + Some(partner) => { + partner.excluded_where_gdpr_applies = true; + true + } + None => false, + } + } + /// Whether an enabled identity provider owns `source`. #[must_use] pub fn is_provider_owned(&self, source: &str) -> bool { @@ -421,6 +450,7 @@ fn build_partner_config( ts_pull_token: partner.ts_pull_token.clone(), provider_owned: false, owner_tcf_vendor_ids: Vec::new(), + excluded_where_gdpr_applies: false, } } diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index 0db3138c8..c815ebaa2 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -66,15 +66,36 @@ pub fn reads_lockr(kind: &str) -> bool { ) } +/// Whether a provider of `kind` excludes `source` where GDPR applies. +fn excludes_where_gdpr_applies(kind: &str, source: &str) -> bool { + match kind { + uid2::Uid2::KIND => uid2::excludes_where_gdpr_applies(source), + liveintent::LiveIntent::KIND => liveintent::excludes_where_gdpr_applies(source), + _ => false, + } +} + /// Builds the partner registry, marking the sources claimed by enabled /// providers as provider-owned. /// +/// The sources any configured provider, enabled or disabled, excludes where +/// GDPR applies are marked too, so the UIDs already stored for them are +/// withheld from egress for a GDPR visitor. +/// /// # Errors /// /// Returns the same errors as [`PartnerRegistry::from_ec_with_kinds`], /// including for an enabled provider whose kind is not registered. pub fn partner_registry(ec: &Ec) -> Result> { - PartnerRegistry::from_ec_with_kinds(ec, is_registered_kind) + let mut registry = PartnerRegistry::from_ec_with_kinds(ec, is_registered_kind)?; + for provider in &ec.identity_providers { + for source in &provider.sources { + if excludes_where_gdpr_applies(provider.kind(), source) { + registry.exclude_where_gdpr_applies(source); + } + } + } + Ok(registry) } /// Builds the provider for a configured `kind`. @@ -562,6 +583,50 @@ mod tests { ); } + #[test] + fn partner_registry_marks_sources_excluded_where_gdpr_applies() { + // Arrange: a disabled `uid2` provider still stored UIDs, and + // `liveintent` resolves UID2 next to its own source. + let mut settings = settings_with_cookie_providers("page", "page"); + for source in ["uidapi.com", "liveintent.example.com"] { + settings.ec.partners.push( + toml::from_str(&format!( + "name = \"{source}\"\nsource_domain = \"{source}\"\nbidstream_enabled = true" + )) + .expect("should parse partner"), + ); + } + settings.ec.identity_providers.push( + toml::from_str( + "id = \"uid2\"\nkind = \"uid2\"\nenabled = false\nsources = [\"uidapi.com\"]", + ) + .expect("should parse uid2"), + ); + settings.ec.identity_providers.push( + toml::from_str( + "id = \"liveintent\"\nkind = \"liveintent\"\nenabled = false\n\ + sources = [\"liveintent.example.com\", \"uidapi.com\"]", + ) + .expect("should parse liveintent"), + ); + + // Act + let registry = partner_registry(&settings.ec).expect("should build partner registry"); + + // Assert + let excluded = |source: &str| { + registry + .get(source) + .is_some_and(|partner| partner.excluded_where_gdpr_applies) + }; + assert!(excluded("uidapi.com"), "should mark the UID2 source"); + assert!( + !excluded("liveintent.example.com"), + "should not mark LiveIntent's own source" + ); + assert!(!excluded("pubcid.org"), "should not mark other sources"); + } + #[test] fn rejects_an_unknown_collect_timing() { toml::from_str::( diff --git a/crates/trusted-server-identity-providers/src/liveintent.rs b/crates/trusted-server-identity-providers/src/liveintent.rs index 86950645d..bac4fd047 100644 --- a/crates/trusted-server-identity-providers/src/liveintent.rs +++ b/crates/trusted-server-identity-providers/src/liveintent.rs @@ -248,6 +248,12 @@ fn expiry(headers: &HeaderMap) -> ProviderExpiry { bounded_expiry(ttl) } +/// Whether a `liveintent` provider excludes `source` where GDPR applies: the +/// UID2 tokens it resolves, matching the `uid2` provider. +pub(crate) fn excludes_where_gdpr_applies(source: &str) -> bool { + source.eq_ignore_ascii_case(UID2_SOURCE) +} + impl IdentityProvider for LiveIntentProvider { fn capabilities(&self) -> &ProviderCapabilities { &self.capabilities @@ -256,7 +262,7 @@ impl IdentityProvider for LiveIntentProvider { /// UID2 tokens `LiveIntent` resolves are not collected or kept where GDPR /// applies, matching the `uid2` provider. fn excluded_where_gdpr_applies(&self, source: &SourceDomain) -> bool { - source.as_str() == UID2_SOURCE + excludes_where_gdpr_applies(source.as_str()) } fn build_request( diff --git a/crates/trusted-server-identity-providers/src/uid2.rs b/crates/trusted-server-identity-providers/src/uid2.rs index 646a53e48..2e4362182 100644 --- a/crates/trusted-server-identity-providers/src/uid2.rs +++ b/crates/trusted-server-identity-providers/src/uid2.rs @@ -276,6 +276,12 @@ fn prebid_tokens(json: &str) -> Option { Some(TokenCookie::Token(identity.advertising_token, expiry)) } +/// Whether a `uid2` provider excludes `source` where GDPR applies: UID2 is +/// not offered there, so every source is excluded. +pub(crate) fn excludes_where_gdpr_applies(_source: &str) -> bool { + true +} + impl IdentityProvider for Uid2Provider { fn capabilities(&self) -> &ProviderCapabilities { &self.capabilities @@ -283,8 +289,8 @@ impl IdentityProvider for Uid2Provider { /// UID2 is not offered where GDPR applies, so none of its sources is /// collected or kept there. - fn excluded_where_gdpr_applies(&self, _source: &SourceDomain) -> bool { - true + fn excluded_where_gdpr_applies(&self, source: &SourceDomain) -> bool { + excludes_where_gdpr_applies(source.as_str()) } /// A server-side refresh does not rewrite the browser's cookies, so a From 339681c8b679d8df92da300e8f3f8dc0d1dbdb10 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 18:29:47 +0530 Subject: [PATCH 67/72] Send the previous ID5 signature back as s Prebid's ID5 module sends the cached reply's signature as s so ID5 can return the same ID. Keep a non-empty reply signature as the UIDs' private refresh state, due a tenth of the TTL before they expire, and send it as s in the next fetch. Signed-off-by: dhruv8sh --- .../src/id5.rs | 156 +++++++++++++++++- .../plans/2026-11-02-identity-provider-id5.md | 4 +- 2 files changed, 152 insertions(+), 8 deletions(-) diff --git a/crates/trusted-server-identity-providers/src/id5.rs b/crates/trusted-server-identity-providers/src/id5.rs index f3723427c..538190981 100644 --- a/crates/trusted-server-identity-providers/src/id5.rs +++ b/crates/trusted-server-identity-providers/src/id5.rs @@ -18,7 +18,10 @@ //! writes (`[settings.lockr]`, default key `id5id`). A cookie holding the ID5 fetch response JSON //! (plain or percent-encoded) is parsed exactly like a lookup response, and //! a bare ID string becomes one UID on the first source with agent type 1 -//! and the default TTL. The response `signature` is never stored. A +//! and the default TTL. A reply's non-empty `signature` is kept as the +//! UIDs' private refresh state, due a tenth of their TTL before they +//! expire, and sent back as `s` in the next fetch so ID5 can return the +//! same ID, as Prebid's ID5 module does. A //! percent-encoded JSON cookie is decoded and its entries, `ext` objects //! included, are copied out of the decoded text. A cookie that yields no //! usable UID is ignored and the lookup runs as usual. A cookie holding @@ -50,8 +53,8 @@ use trusted_server_core::ec::eid_types::{SourceDomain, UnixSeconds}; use trusted_server_core::ec::provider::{ CookieRead, EidLevelFields, EidShape, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, ProviderConfigError, ProviderEid, ProviderError, - ProviderKind, ProviderRequest, ProviderResponse, ProviderUid, RequestCookies, SourceCapability, - SourceInput, + ProviderExpiry, ProviderKind, ProviderRefresh, ProviderRequest, ProviderResponse, ProviderUid, + RequestCookies, SourceCapability, SourceInput, }; use url::Url; @@ -200,6 +203,9 @@ struct FetchBody<'a> { ip: Option, #[serde(skip_serializing_if = "Option::is_none")] ipv6: Option, + /// The previous reply's `signature`, which lets ID5 return the same ID. + #[serde(skip_serializing_if = "Option::is_none")] + s: Option<&'a str>, } /// The top-level reply; each field is parsed separately so one bad shape @@ -214,6 +220,8 @@ struct FetchReply<'body> { cache_control: Option<&'body RawValue>, #[serde(default, borrow)] ext: Option<&'body RawValue>, + #[serde(default, borrow)] + signature: Option<&'body RawValue>, } #[derive(Deserialize)] @@ -261,6 +269,16 @@ impl Id5Provider { .and_then(|cache| cache.max_age_sec) .map(Duration::from_secs); let expiry = bounded_expiry(max_age); + let signature = reply + .signature + .and_then(|raw| serde_json::from_str::>(raw.get()).ok()) + .filter(|signature| !signature.is_empty()); + let refresh = || { + signature.as_ref().map(|signature| ProviderRefresh { + state: signature.clone(), + from: refresh_from(expiry), + }) + }; let entries: BTreeMap = reply .ids @@ -277,6 +295,9 @@ impl Id5Provider { (!eid.uids.is_empty()).then(|| eid.into_provider_eid(expiry)) }) .collect(); + for uid in eids.iter_mut().flat_map(|eid| eid.uids.iter_mut()) { + uid.refresh = refresh(); + } let primary = self.primary_source().as_str(); let has_primary = eids.iter().any(|eid| eid.source == primary); let universal_uid = reply @@ -302,7 +323,7 @@ impl Id5Provider { atype: Some(1), ext: link_type.map(Cow::Owned), expiry, - refresh: None, + refresh: refresh(), }], }); } @@ -375,6 +396,12 @@ impl IdentityProvider for Id5Provider { ua: lookup.client.user_agent, ip: routable_ip.filter(IpAddr::is_ipv4).map(|ip| ip.to_string()), ipv6: routable_ip.filter(IpAddr::is_ipv6).map(|ip| ip.to_string()), + s: lookup + .current + .iter() + .flat_map(|(_, eid)| eid.uids()) + .find_map(|uid| uid.refresh()) + .map(|refresh| refresh.state.as_str()), }; let body = serde_json::to_vec(&body).change_context(ProviderError::Request)?; let mut headers = HeaderMap::new(); @@ -467,12 +494,25 @@ fn owned<'any>(eid: ProviderEid<'_>) -> ProviderEid<'any> { atype: uid.atype, ext: uid.ext.map(|ext| Cow::Owned(ext.into_owned())), expiry: uid.expiry, - refresh: None, + refresh: uid.refresh.map(|refresh| ProviderRefresh { + state: Cow::Owned(refresh.state.into_owned()), + from: refresh.from, + }), }) .collect(), } } +/// When a UID that expires at `expiry` is refreshed: a tenth of its TTL +/// early, so the refresh runs while the UID and its signature are still +/// stored. +fn refresh_from(expiry: ProviderExpiry) -> ProviderExpiry { + match expiry { + ProviderExpiry::After(ttl) => ProviderExpiry::After(ttl - ttl / 10), + other => other, + } +} + /// Returns whether `id` is a real UID: not empty and not the opt-out UID. fn is_usable_uid(id: &str) -> bool { !id.is_empty() && id != OPTED_OUT_UID @@ -483,6 +523,8 @@ mod tests { use http::StatusCode; use serde_json::json; use trusted_server_core::consent::types::ConsentContext; + use trusted_server_core::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, UidValue}; + use trusted_server_core::ec::kv_types::{KvEid, KvEntry, KvPartnerId, KvRefresh, KvUid}; use trusted_server_core::ec::provider::{ ClientSignals, OwnedEntries, ProviderExpiry, build_kind, }; @@ -518,6 +560,108 @@ mod tests { .expect("should parse response") } + #[test] + fn keeps_the_reply_signature_as_refresh_state() { + let provider = provider(); + let headers = HeaderMap::new(); + let body = json!({ + "universal_uid": "id5-example", + "signature": "signature-example", + "cache_control": { "max_age_sec": 36_000 }, + }) + .to_string(); + + let outcome = respond(&*provider, &headers, body.as_bytes()); + + let LookupOutcome::Found(eids) = outcome else { + panic!("should find the ID5 ID"); + }; + let refresh = eids[0].uids[0] + .refresh + .as_ref() + .expect("should keep the signature as refresh state"); + assert_eq!( + refresh.state, "signature-example", + "should store the signature" + ); + assert!( + matches!(refresh.from, ProviderExpiry::After(ttl) if ttl == Duration::from_secs(32_400)), + "should refresh a tenth of the TTL before the UID expires" + ); + } + + #[test] + fn keeps_no_refresh_state_without_a_signature() { + let provider = provider(); + let headers = HeaderMap::new(); + let body = json!({ "universal_uid": "id5-example", "signature": "" }).to_string(); + + let outcome = respond(&*provider, &headers, body.as_bytes()); + + let LookupOutcome::Found(eids) = outcome else { + panic!("should find the ID5 ID"); + }; + assert!( + eids[0].uids[0].refresh.is_none(), + "should keep no refresh state for an empty signature" + ); + } + + #[test] + fn sends_the_stored_signature_as_s() { + // Arrange + let uid = KvUid::new( + UidValue::try_from("id5-example".to_owned()).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("id5")), + Expiry::At(UnixSeconds::new(1_800_000_000)), + ) + .with_refresh(KvRefresh { + state: "signature-example".to_owned(), + from: UnixSeconds::new(1_700_000_000), + }); + let record = KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![uid])], + UnixSeconds::new(1_700_000_000), + ) + .expect("should keep the live uid"); + let mut entry = KvEntry::tombstone(1); + entry.consent.ok = true; + entry.ids.insert("id5.example.com".to_owned(), record); + let sources = [SourceDomain::try_from("id5.example.com").expect("should build source")]; + let owner = ProviderId::new("id5"); + let consent = ConsentContext::default(); + let lookup = LookupInput { + ec_id: "ec-example", + consent: &consent, + client: ClientSignals { + ip: None, + user_agent: None, + }, + publisher_domain: "www.example.com", + due_sources: &sources, + current: OwnedEntries::new(Some(&entry), &sources, &owner), + cookies: RequestCookies::default(), + hashed_email: None, + now: UnixSeconds::new(1_700_000_000), + }; + + // Act + let request = provider() + .build_request(&lookup) + .expect("should build request"); + + // Assert + let body: serde_json::Value = + serde_json::from_slice(&request.body).expect("should send JSON"); + assert_eq!( + body.get("s"), + Some(&json!("signature-example")), + "should send the previous signature as s" + ); + } + #[test] fn posts_consent_and_network_signals_to_the_partner_url() { let provider = provider(); @@ -867,7 +1011,7 @@ mod tests { true, Duration::from_secs(7200) )], - "should parse the reply like a lookup response and never store the signature" + "should parse the reply like a lookup response" ); } diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-id5.md b/docs/superpowers/plans/2026-11-02-identity-provider-id5.md index e29b1c51a..a4d741ccb 100644 --- a/docs/superpowers/plans/2026-11-02-identity-provider-id5.md +++ b/docs/superpowers/plans/2026-11-02-identity-provider-id5.md @@ -13,7 +13,7 @@ **Changes from the earlier ID5 plan (built on the old descriptor framework):** - No hashed email and no `pd` field: every source is `SourceInput::Anonymous`. -- No private provider state, so ID5's `signature` is not sent back. +- A reply's non-empty `signature` is kept as private refresh state, due a tenth of the TTL before the UIDs expire, and sent back as `s`, as Prebid's ID5 module does. - Endpoint, sources and the TCF vendor ID come from the common provider config, not hard-coded defaults. - The legacy `universal_uid` fallback goes to the first configured source. An opted-out `universal_uid` of `0` is reported as `LookupOutcome::OptedOut`, detected on the reply as sent (not percent-decoded). @@ -38,7 +38,7 @@ ID5's source is provider-owned, so the browser `ts-eids` copy is dropped. Prebid - `id5` declares the Lockr capability (see `2026-11-02-identity-provider-lockr-capability.md`): with `[settings.lockr] enabled = true`, the Lockr cookies (default `["id5id"]`) are read after `cookie_names` and before the lookup, with the same parser. - A value that is JSON after percent-decoding is parsed with the same code as a lookup response (lenient entries, opt-out `0` dropped, normalized sources, `cache_control.max_age_sec` bounded). When the value was percent-encoded, entries, `ext` objects included, are copied out of the decoded text. - Any other value is a bare ID: one UID on the first source with `atype = 1` and the default TTL. -- `signature` is never stored as a UID. A cookie that yields no usable UID is ignored, and the lookup runs as usual; a kept UID skips the lookup. +- `signature` is never stored as a UID, only as refresh state. A cookie that yields no usable UID is ignored, and the lookup runs as usual; a kept UID skips the lookup. ## Tasks From f364d970899bf365cf62e590dec4fba821ebd0d4 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 18:59:35 +0530 Subject: [PATCH 68/72] Withhold browser-sent EIDs of GDPR-excluded sources from bidders The GDPR source exclusion applied only to stored rows, so a UID2 EID the browser sent in the /auction body or the ts-eids cookie still reached bidders for a GDPR visitor. drop_vendor_denied_client_eids now also drops client EIDs whose source a provider excludes where GDPR applies, which covers /auction and the server-side auction. Signed-off-by: dhruv8sh --- .../src/auction/endpoints.rs | 53 ++++++++++++++++--- 1 file changed, 47 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 38f43d607..310b41bf9 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -639,10 +639,11 @@ fn parse_client_auction_uid(raw: &JsonValue) -> Option { } /// Drops client EIDs whose source is owned by a configured identity provider -/// whose TCF vendor consent is withheld. +/// whose TCF vendor consent is withheld, and, for a visitor whose jurisdiction +/// is GDPR, client EIDs whose source a provider excludes where GDPR applies. /// -/// Only sources marked by a configured provider carry a vendor ID, so with no -/// providers configured the input is returned unchanged. +/// Only sources marked by a configured provider carry a vendor ID or a GDPR +/// exclusion, so with no providers configured the input is returned unchanged. pub(crate) fn drop_vendor_denied_client_eids( client_eids: Option>, registry: Option<&PartnerRegistry>, @@ -653,9 +654,9 @@ pub(crate) fn drop_vendor_denied_client_eids( }; let mut eids = client_eids?; eids.retain(|eid| { - !registry - .get(&eid.source) - .is_some_and(|partner| partner.owner_vendor_consent_denied(consent)) + !registry.get(&eid.source).is_some_and(|partner| { + partner.owner_vendor_consent_denied(consent) || partner.excluded_for(consent) + }) }); Some(eids) } @@ -879,6 +880,46 @@ mod tests { ); } + #[test] + fn client_eids_of_a_gdpr_excluded_source_are_dropped_where_gdpr_applies() { + // Arrange: the browser sends a UID2 EID; the visitor is under GDPR + // with TCF Purposes 1 and 4 granted. + let mut registry = partner_registry(&["uidapi.com", "ids.example.com"]); + assert!( + registry.exclude_where_gdpr_applies("uidapi.com"), + "should mark the UID2 source" + ); + let gdpr = crate::test_support::tests::consent_granting_vendors(vec![1]); + + // Act + let under_gdpr = drop_vendor_denied_client_eids( + client_eids_for(&["uidapi.com", "ids.example.com"]), + Some(®istry), + &gdpr, + ) + .expect("should keep the list"); + let elsewhere = drop_vendor_denied_client_eids( + client_eids_for(&["uidapi.com", "ids.example.com"]), + Some(®istry), + &ConsentContext::default(), + ) + .expect("should keep the list"); + + // Assert + let sources = + |eids: &[Eid]| -> Vec { eids.iter().map(|eid| eid.source.clone()).collect() }; + assert_eq!( + sources(&under_gdpr), + vec!["ids.example.com"], + "should not send the GDPR-excluded browser EID to bidders" + ); + assert_eq!( + sources(&elsewhere), + vec!["uidapi.com", "ids.example.com"], + "should keep the browser EID outside GDPR" + ); + } + #[test] fn client_eids_are_unchanged_without_providers() { // Arrange From 1427dc421bf2a36b193bf1c9b09cb32b4289c8e2 Mon Sep 17 00:00:00 2001 From: Dhruvesh Surolia Date: Thu, 12 Nov 2026 19:31:32 +0530 Subject: [PATCH 69/72] Potential fix for pull request finding 'CodeQL / Cleartext logging of sensitive information' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- crates/trusted-server-identity-providers/src/connectid.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/trusted-server-identity-providers/src/connectid.rs b/crates/trusted-server-identity-providers/src/connectid.rs index e33fe75cd..e1be4442f 100644 --- a/crates/trusted-server-identity-providers/src/connectid.rs +++ b/crates/trusted-server-identity-providers/src/connectid.rs @@ -198,7 +198,7 @@ mod tests { fn expires_at(read: Option<(String, ProviderExpiry)>) -> u64 { match read.expect("should read the ID").1 { ProviderExpiry::At(at) => at.get(), - other => panic!("should expire at a time, got {other:?}"), + _ => panic!("should expire at a time"), } } From ea8d6245df56a6ec727aa52ac382e9b12c476749 Mon Sep 17 00:00:00 2001 From: Dhruvesh Surolia Date: Thu, 12 Nov 2026 19:31:46 +0530 Subject: [PATCH 70/72] Potential fix for pull request finding 'CodeQL / Hard-coded cryptographic value' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- crates/trusted-server-identity-providers/src/uid2.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/crates/trusted-server-identity-providers/src/uid2.rs b/crates/trusted-server-identity-providers/src/uid2.rs index 2e4362182..5ea1fbda0 100644 --- a/crates/trusted-server-identity-providers/src/uid2.rs +++ b/crates/trusted-server-identity-providers/src/uid2.rs @@ -42,7 +42,7 @@ use std::borrow::Cow; -use aes_gcm::aead::{Aead as _, KeyInit as _}; +use aes_gcm::aead::{Aead as _, KeyInit as _, OsRng}; use aes_gcm::{Aes256Gcm, Nonce}; use base64::Engine as _; use base64::engine::general_purpose::STANDARD as BASE64; @@ -874,8 +874,9 @@ mod tests { ); } - fn encrypted_reply(reply: &serde_json::Value) -> String { - let cipher = Aes256Gcm::new_from_slice(&[7_u8; 32]).expect("should build cipher"); + fn encrypted_reply(reply: &serde_json::Value) -> (String, String) { + let key = Aes256Gcm::generate_key(&mut OsRng); + let cipher = Aes256Gcm::new(&key); let iv = [1_u8; IV_BYTES]; let ciphertext = cipher .encrypt( @@ -883,7 +884,10 @@ mod tests { reply.to_string().as_bytes(), ) .expect("should encrypt"); - BASE64.encode([&iv[..], &ciphertext].concat()) + ( + BASE64.encode([&iv[..], &ciphertext].concat()), + BASE64.encode(key.as_slice()), + ) } fn reply<'a>(body: &'a str, key: &'a str, headers: &'a HeaderMap) -> ProviderResponse<'a> { From 4fd9bd8d970a590c455cc27d4bfb12e637efd4b7 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 19:35:45 +0530 Subject: [PATCH 71/72] Restore the fixed test key in the UID2 reply tests The Copilot Autofix change generated the key with aes_gcm::aead::OsRng, which this aes-gcm build does not export, so the crate stopped compiling. A fixed key in test-only code is intended. Signed-off-by: dhruv8sh --- crates/trusted-server-identity-providers/src/uid2.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/crates/trusted-server-identity-providers/src/uid2.rs b/crates/trusted-server-identity-providers/src/uid2.rs index 5ea1fbda0..2e4362182 100644 --- a/crates/trusted-server-identity-providers/src/uid2.rs +++ b/crates/trusted-server-identity-providers/src/uid2.rs @@ -42,7 +42,7 @@ use std::borrow::Cow; -use aes_gcm::aead::{Aead as _, KeyInit as _, OsRng}; +use aes_gcm::aead::{Aead as _, KeyInit as _}; use aes_gcm::{Aes256Gcm, Nonce}; use base64::Engine as _; use base64::engine::general_purpose::STANDARD as BASE64; @@ -874,9 +874,8 @@ mod tests { ); } - fn encrypted_reply(reply: &serde_json::Value) -> (String, String) { - let key = Aes256Gcm::generate_key(&mut OsRng); - let cipher = Aes256Gcm::new(&key); + fn encrypted_reply(reply: &serde_json::Value) -> String { + let cipher = Aes256Gcm::new_from_slice(&[7_u8; 32]).expect("should build cipher"); let iv = [1_u8; IV_BYTES]; let ciphertext = cipher .encrypt( @@ -884,10 +883,7 @@ mod tests { reply.to_string().as_bytes(), ) .expect("should encrypt"); - ( - BASE64.encode([&iv[..], &ciphertext].concat()), - BASE64.encode(key.as_slice()), - ) + BASE64.encode([&iv[..], &ciphertext].concat()) } fn reply<'a>(body: &'a str, key: &'a str, headers: &'a HeaderMap) -> ProviderResponse<'a> { From 7f55776f3c199da165f3bc54533a674c799497e1 Mon Sep 17 00:00:00 2001 From: dhruv8sh Date: Thu, 12 Nov 2026 20:11:29 +0530 Subject: [PATCH 72/72] Generate the UID2 refresh test key at runtime The encrypted_reply test helper built its AES-256-GCM key from a constant, which CodeQL flags as a hard-coded cryptographic value. Generate the key from the system RNG instead and return it alongside the body so the decrypting tests pass it as the reply context. The getrandom feature is enabled on aes-gcm for dev builds only. Signed-off-by: dhruv8sh --- Cargo.lock | 2 ++ .../Cargo.toml | 2 ++ .../src/uid2.rs | 31 +++++++++++++------ 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 66e402766..093280867 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1108,7 +1108,9 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ + "getrandom 0.4.3", "hybrid-array", + "rand_core 0.10.1", ] [[package]] diff --git a/crates/trusted-server-identity-providers/Cargo.toml b/crates/trusted-server-identity-providers/Cargo.toml index d267ebc66..d3bd486f7 100644 --- a/crates/trusted-server-identity-providers/Cargo.toml +++ b/crates/trusted-server-identity-providers/Cargo.toml @@ -30,4 +30,6 @@ urlencoding = { workspace = true } uuid = { workspace = true } [dev-dependencies] +# Test-only key generation from the system RNG. +aes-gcm = { workspace = true, features = ["getrandom"] } toml = { workspace = true } diff --git a/crates/trusted-server-identity-providers/src/uid2.rs b/crates/trusted-server-identity-providers/src/uid2.rs index 2e4362182..03da983d6 100644 --- a/crates/trusted-server-identity-providers/src/uid2.rs +++ b/crates/trusted-server-identity-providers/src/uid2.rs @@ -423,6 +423,8 @@ impl Uid2Provider { #[cfg(test)] mod tests { + use aes_gcm::Key; + use aes_gcm::aead::Generate as _; use serde_json::json; use trusted_server_core::ec::provider::build_kind; use trusted_server_core::settings::{IdentityProviderConfig, ProviderSecrets}; @@ -874,8 +876,16 @@ mod tests { ); } - fn encrypted_reply(reply: &serde_json::Value) -> String { - let cipher = Aes256Gcm::new_from_slice(&[7_u8; 32]).expect("should build cipher"); + fn random_key() -> String { + BASE64.encode(Key::::generate()) + } + + /// Encrypts `reply` under a freshly generated key. + /// + /// Returns the base64 body and the base64 key that decrypts it. + fn encrypted_reply(reply: &serde_json::Value) -> (String, String) { + let key = Key::::generate(); + let cipher = Aes256Gcm::new(&key); let iv = [1_u8; IV_BYTES]; let ciphertext = cipher .encrypt( @@ -883,7 +893,10 @@ mod tests { reply.to_string().as_bytes(), ) .expect("should encrypt"); - BASE64.encode([&iv[..], &ciphertext].concat()) + ( + BASE64.encode([&iv[..], &ciphertext].concat()), + BASE64.encode(key), + ) } fn reply<'a>(body: &'a str, key: &'a str, headers: &'a HeaderMap) -> ProviderResponse<'a> { @@ -897,7 +910,7 @@ mod tests { #[test] fn decrypts_a_refreshed_identity() { - let body = encrypted_reply(&json!({ + let (body, key) = encrypted_reply(&json!({ "status": "success", "body": { "advertising_token": "AgAAAArefreshedtoken", @@ -909,7 +922,7 @@ mod tests { }, })); - let (key, headers) = (BASE64.encode([7_u8; 32]), HeaderMap::new()); + let headers = HeaderMap::new(); let outcome = refreshing_provider() .parse_response(reply(&body, &key, &headers)) @@ -943,9 +956,9 @@ mod tests { #[test] fn reports_an_opt_out_from_the_refresh() { - let body = encrypted_reply(&json!({ "status": "optout" })); + let (body, key) = encrypted_reply(&json!({ "status": "optout" })); - let (key, headers) = (BASE64.encode([7_u8; 32]), HeaderMap::new()); + let headers = HeaderMap::new(); let outcome = refreshing_provider() .parse_response(reply(&body, &key, &headers)) @@ -959,9 +972,9 @@ mod tests { #[test] fn rejects_a_reply_under_another_key() { - let body = encrypted_reply(&json!({ "status": "optout" })); + let (body, _) = encrypted_reply(&json!({ "status": "optout" })); - let (key, headers) = (BASE64.encode([8_u8; 32]), HeaderMap::new()); + let (key, headers) = (random_key(), HeaderMap::new()); assert!( refreshing_provider()