diff --git a/Cargo.lock b/Cargo.lock index d9cf35f9f..66e402766 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,6 +18,31 @@ dependencies = [ "inout", ] +[[package]] +name = "aes" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fc36d7440d015b45468ba08a21e80e056ebbd014c611c6298bb7503e04bd1" +dependencies = [ + "cipher", + "cpubits", + "cpufeatures 0.3.0", +] + +[[package]] +name = "aes-gcm" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2b8006a0c83f52b62ba44a97b58bf76fe2f70a329e588f67f89691d93d498f" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ctutils", + "ghash", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -976,6 +1001,12 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -1126,6 +1157,15 @@ dependencies = [ "linktime-proc-macro", ] +[[package]] +name = "ctr" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" +dependencies = [ + "cipher", +] + [[package]] name = "ctutils" version = "0.4.3" @@ -2214,6 +2254,15 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "ghash" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" +dependencies = [ + "polyval", +] + [[package]] name = "gix" version = "0.83.0" @@ -2594,7 +2643,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c91d8cffac8849493a82233811bd02b2b183b8cf39bf704de0fa0841b737595" dependencies = [ "bstr", - "hashbrown 0.16.1", + "hashbrown 0.17.1", ] [[package]] @@ -4629,6 +4678,17 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "polyval" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" +dependencies = [ + "cpubits", + "cpufeatures 0.3.0", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.13.1" @@ -4790,7 +4850,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck", - "itertools 0.10.5", + "itertools 0.13.0", "log", "multimap", "once_cell", @@ -4810,7 +4870,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.118", @@ -4823,7 +4883,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.118", @@ -6679,8 +6739,10 @@ dependencies = [ "tracing", "trusted-server-adapter-axum", "trusted-server-core", + "trusted-server-identity-providers", "url", "uuid", + "validator", "webpki-roots", "which", "x509-parser", @@ -6742,9 +6804,21 @@ dependencies = [ name = "trusted-server-identity-providers" version = "0.1.0" dependencies = [ + "aes-gcm", + "base64", + "bytes", + "chrono", "error-stack", + "http", + "httpdate", + "log", + "serde", + "serde_json", "toml", "trusted-server-core", + "url", + "urlencoding", + "uuid", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f43101230..0c5542a45 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ publish = false version = "0.1.0" [workspace.dependencies] +aes-gcm = { version = "0.11", default-features = false, features = ["aes", "alloc"] } anyhow = "1" assert_cmd = "2" async-stream = "0.3" diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index fa04bf437..4a5f38e66 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -122,7 +122,7 @@ use trusted_server_core::ec::eid_types::UnixSeconds; use trusted_server_core::ec::identify::{cors_preflight_identify, handle_identify}; use trusted_server_core::ec::kv::KvIdentityGraph; use trusted_server_core::ec::page_load::NavigationPermit; -use trusted_server_core::ec::provider::ClientSignals; +use trusted_server_core::ec::provider::{ClientSignals, RequestCookies}; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::ec::{EcContext, EcKvSnapshot, EidSyncSource}; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; @@ -395,6 +395,7 @@ pub(crate) struct AfterResponseLookups { pub(crate) ec_id: String, pub(crate) client_ip: Option, pub(crate) user_agent: Option, + pub(crate) cookie_header: String, } /// Identity work a navigation hands to the entry point. @@ -427,6 +428,7 @@ pub(crate) fn take_navigation_lookups() -> NavigationLookups { struct ProviderRequestSignals<'a> { client_ip: Option, user_agent: Option<&'a str>, + cookie_header: String, } impl<'a> ProviderRequestSignals<'a> { @@ -439,10 +441,23 @@ impl<'a> ProviderRequestSignals<'a> { .headers() .get(header::USER_AGENT) .and_then(|value| value.to_str().ok()), + cookie_header: joined_cookie_header(req), } } } +/// Joins every `Cookie` header on the request into one `; `-separated value. +/// +/// HTTP/2 and HTTP/3 clients may split cookies across several headers. +fn joined_cookie_header(req: &Request) -> String { + req.headers() + .get_all(header::COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .collect::>() + .join("; ") +} + /// Captures the request signals for after-response identity lookups on a /// permitted publisher navigation. /// @@ -461,6 +476,7 @@ fn prepare_after_response_lookups( ec_id, client_ip: signals.client_ip, user_agent: signals.user_agent.map(str::to_owned), + cookie_header: signals.cookie_header, }) } @@ -502,6 +518,7 @@ async fn dispatch_identity_lookups( user_agent: signals.user_agent, }, publisher_domain: &state.settings.publisher.domain, + cookies: RequestCookies::new(&signals.cookie_header), rate_limiter: &limiter, now: UnixSeconds::new(now), }, @@ -1630,7 +1647,7 @@ mod tests { use trusted_server_core::ec::provider::{ EidLevelFields, EidShape, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderEid, ProviderError, ProviderExpiry, ProviderRequest, - ProviderResponse, ProviderUid, SourceCapability, SourceInput, + ProviderResponse, ProviderUid, RequestCookies, SourceCapability, SourceInput, }; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::TrustedServerError; @@ -2799,6 +2816,29 @@ mod tests { request } + #[test] + fn provider_cookies_join_every_cookie_header() { + // Arrange: HTTP/2 clients may split cookies across headers. + let mut request = navigation_request("/article"); + let headers = request.headers_mut(); + headers.append(header::COOKIE, HeaderValue::from_static("a=1; b=2")); + headers.append(header::COOKIE, HeaderValue::from_static("vendor_id=c-1")); + + // Act + let joined = super::joined_cookie_header(&request); + + // Assert + assert_eq!( + joined, "a=1; b=2; vendor_id=c-1", + "should join every Cookie header" + ); + assert_eq!( + RequestCookies::new(&joined).get("vendor_id"), + Some("c-1"), + "should let providers read a cookie from a later header" + ); + } + fn take_finalize_state(response: &mut Response) -> super::EcFinalizeState { response .extensions_mut() diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 6aafcc9c4..f2aff9de1 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -25,7 +25,7 @@ use trusted_server_core::ec::eid_types::UnixSeconds; use trusted_server_core::ec::finalize::ec_finalize_response; use trusted_server_core::ec::kv::KvIdentityGraph; use trusted_server_core::ec::page_load::NavigationPermit; -use trusted_server_core::ec::provider::ClientSignals; +use trusted_server_core::ec::provider::{ClientSignals, RequestCookies}; use trusted_server_core::ec::pull_sync::{ PullSyncContext, build_pull_sync_context, dispatch_pull_sync, }; @@ -801,6 +801,7 @@ pub(crate) fn run_after_response_lookups( user_agent: lookups.user_agent.as_deref(), }, publisher_domain: &settings.publisher.domain, + cookies: RequestCookies::new(&lookups.cookie_header), rate_limiter: &limiter, now: UnixSeconds::new(now), }, diff --git a/crates/trusted-server-cli/Cargo.toml b/crates/trusted-server-cli/Cargo.toml index 4d6af83fd..70b85f497 100644 --- a/crates/trusted-server-cli/Cargo.toml +++ b/crates/trusted-server-cli/Cargo.toml @@ -57,8 +57,10 @@ toml = { workspace = true } toml_edit = { workspace = true } tracing = { workspace = true } trusted-server-core = { workspace = true } +trusted-server-identity-providers = { workspace = true } url = { workspace = true } uuid = { workspace = true } +validator = { workspace = true } which = { workspace = true } # `ts dev lint domains` and `ts dev install-hooks` drive all git access through diff --git a/crates/trusted-server-cli/src/checked_app_config.rs b/crates/trusted-server-cli/src/checked_app_config.rs new file mode 100644 index 000000000..bc51ecbf0 --- /dev/null +++ b/crates/trusted-server-cli/src/checked_app_config.rs @@ -0,0 +1,170 @@ +//! The app-config type that `ts config push`, `validate` and `diff` check. +//! +//! [`TrustedServerAppConfig`] validation lives in core, which cannot build +//! identity providers because the provider crate depends on core. At runtime +//! a provider that fails to build disables every provider for the request, so +//! this wrapper adds [`validate_providers`] to the config checks: a +//! provider-specific setting error then fails the command before deploy. + +use std::borrow::Cow; + +use edgezero_core::app_config::{AppConfigMeta, SecretField}; +use error_stack::{AttachmentKind, FrameKind, Report}; +use serde::{Deserialize, Serialize}; +use trusted_server_core::config::TrustedServerAppConfig; +use trusted_server_core::ec::provider::ProviderConfigError; +use trusted_server_identity_providers::validate_providers; +use validator::{Validate, ValidationError, ValidationErrors}; + +/// Field that identity-provider build errors are reported under. +const PROVIDER_VALIDATION_FIELD: &str = "identity_providers"; + +/// [`TrustedServerAppConfig`] whose validation also builds every enabled +/// identity provider. +#[derive(Serialize, Deserialize)] +#[serde(transparent)] +pub(crate) struct CheckedAppConfig(TrustedServerAppConfig); + +impl Validate for CheckedAppConfig { + fn validate(&self) -> Result<(), ValidationErrors> { + let mut errors = self.0.validate().err().unwrap_or_default(); + if let Err(report) = validate_providers(self.0.settings()) { + let mut error = ValidationError::new("trusted_server_identity_providers"); + error.message = Some(Cow::Owned(provider_error_message(&report))); + errors.add(PROVIDER_VALIDATION_FIELD, error); + } + if errors.errors().is_empty() { + Ok(()) + } else { + Err(errors) + } + } +} + +impl AppConfigMeta for CheckedAppConfig { + fn secret_fields() -> Vec { + TrustedServerAppConfig::secret_fields() + } +} + +/// Joins the report's context with every printable attachment and underlying +/// error, which name the failing provider and setting (for example the +/// settings parser's `unknown field` error). +fn provider_error_message(report: &Report) -> String { + let mut details: Vec = Vec::new(); + for frame in report.frames() { + let detail = match frame.kind() { + FrameKind::Attachment(AttachmentKind::Printable(printable)) => printable.to_string(), + FrameKind::Context(context) if !frame.is::() => { + context.to_string() + } + _ => continue, + }; + if !details.contains(&detail) { + details.push(detail); + } + } + if details.is_empty() { + report.current_context().to_string() + } else { + format!("{}: {}", report.current_context(), details.join("; ")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const EXAMPLE_CONFIG: &str = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../trusted-server.example.toml" + )); + + fn with_liveramp(settings: &str) -> CheckedAppConfig { + let config = format!( + "{EXAMPLE_CONFIG}\n\ + [[ec.partners]]\n\ + name = \"liveramp\"\n\ + source_domain = \"liveramp.example.com\"\n\ + bidstream_enabled = true\n\ + \n\ + [[ec.identity_providers]]\n\ + id = \"liveramp\"\n\ + kind = \"liveramp\"\n\ + endpoint = \"https://envelope.example.com/api/identity/v2/envelope\"\n\ + allowed_domains = [\"envelope.example.com\"]\n\ + sources = [\"liveramp.example.com\"]\n\ + [ec.identity_providers.settings]\n\ + {settings}\n" + ); + toml::from_str(&config).expect("should parse app config") + } + + /// The identity-provider validation message, when there is one. + /// + /// The example config fails unrelated core checks (placeholder publisher + /// values, unresolved secrets), so tests look only at this field. + fn provider_error(config: &CheckedAppConfig) -> Option { + let errors = config.validate().err()?; + errors + .field_errors() + .get(PROVIDER_VALIDATION_FIELD) + .and_then(|errors| errors.first()) + .and_then(|error| error.message.as_deref()) + .map(str::to_owned) + } + + #[test] + fn reports_no_provider_error_for_the_example_config() { + let config: CheckedAppConfig = + toml::from_str(EXAMPLE_CONFIG).expect("should parse the example config"); + + assert_eq!( + provider_error(&config), + None, + "should build the example config's providers" + ); + } + + #[test] + fn reports_no_provider_error_for_a_provider_that_builds() { + let config = with_liveramp("placement_id = 14\ncustom_id_namespace = \"ana-example\""); + + assert_eq!( + provider_error(&config), + None, + "should accept a complete liveramp lookup" + ); + } + + #[test] + fn rejects_a_provider_that_cannot_be_built_and_keeps_core_errors() { + let config = with_liveramp("placement_id = 14"); + + let message = provider_error(&config).expect("should report the provider error"); + let errors = config.validate().expect_err("should fail validation"); + + assert!( + message.contains("custom_id_namespace"), + "should name the missing setting, got: {message}" + ); + assert!( + errors.errors().contains_key("trusted_server"), + "should keep the core validation errors" + ); + } + + #[test] + fn names_the_unknown_setting_in_a_provider_settings_typo() { + let config = with_liveramp( + "placement_id = 14\ncustom_id_namespace = \"ana-example\"\ncookie_nmes = [\"_lr_env\"]", + ); + + let message = provider_error(&config).expect("should report the provider error"); + + assert!( + message.contains("cookie_nmes"), + "should name the unknown setting, got: {message}" + ); + } +} diff --git a/crates/trusted-server-cli/src/lib.rs b/crates/trusted-server-cli/src/lib.rs index 7f6b9e6e7..60dd54ce8 100644 --- a/crates/trusted-server-cli/src/lib.rs +++ b/crates/trusted-server-cli/src/lib.rs @@ -3,6 +3,8 @@ mod ad_templates; #[cfg(not(target_arch = "wasm32"))] mod app_config; #[cfg(not(target_arch = "wasm32"))] +mod checked_app_config; +#[cfg(not(target_arch = "wasm32"))] mod error; #[cfg(not(target_arch = "wasm32"))] mod fs_util; diff --git a/crates/trusted-server-cli/src/run.rs b/crates/trusted-server-cli/src/run.rs index fa91e0a44..d884aef78 100644 --- a/crates/trusted-server-cli/src/run.rs +++ b/crates/trusted-server-cli/src/run.rs @@ -5,8 +5,8 @@ use edgezero_cli::args::{ ActiveVersionArgs, AuthArgs, BuildArgs, ConfigDiffArgs, ConfigGcArgs, ConfigPushArgs, ConfigValidateArgs, DeployArgs, HealthcheckArgs, ProvisionArgs, RollbackArgs, ServeArgs, }; -use trusted_server_core::config::TrustedServerAppConfig; +use crate::checked_app_config::CheckedAppConfig; use crate::commands::audit::{AuditArgs, run_audit}; use crate::commands::config::ad_templates::{AdTemplatesCommand, run_ad_templates}; use crate::commands::config::init::{ConfigInitArgs, run_config_init}; @@ -130,7 +130,7 @@ fn dispatch(args: Args) -> Result { run_config_init(&args).map(|()| RunOutcome::Success) } Command::Config(ConfigCommand::Diff(args)) => { - match edgezero_cli::run_config_diff_typed::(&args) { + match edgezero_cli::run_config_diff_typed::(&args) { Ok(edgezero_cli::DiffExit { code: 0 }) => Ok(RunOutcome::Success), Ok(edgezero_cli::DiffExit { code: 1 }) => Ok(RunOutcome::AssertionFailed), Ok(edgezero_cli::DiffExit { code }) => process::exit(code), @@ -141,11 +141,11 @@ fn dispatch(args: Args) -> Result { edgezero_cli::run_config_gc(&args).map(|()| RunOutcome::Success) } Command::Config(ConfigCommand::Push(args)) => { - edgezero_cli::run_config_push_typed::(&args) + edgezero_cli::run_config_push_typed::(&args) .map(|()| RunOutcome::Success) } Command::Config(ConfigCommand::Validate(args)) => { - edgezero_cli::run_config_validate_typed::(&args) + edgezero_cli::run_config_validate_typed::(&args) .map(|()| RunOutcome::Success) } Command::Deploy(args) => edgezero_cli::run_deploy(&args).map(|()| RunOutcome::Success), diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 1155a786d..38f43d607 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -2054,6 +2054,46 @@ mod tests { ); } + #[test] + fn resolve_auction_eids_withholds_a_gdpr_excluded_source_where_gdpr_applies() { + // Arrange: the row holds a UID2 UID written while the visitor was + // outside GDPR; the visitor now arrives under GDPR with TCF Purposes + // 1 and 4 granted. + let mut registry = partner_registry(&["uidapi.com", "ids.example.com"]); + assert!( + registry.exclude_where_gdpr_applies("uidapi.com"), + "should mark the UID2 source" + ); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let mut entry = crate::ec::kv_types::KvEntry::minimal("uidapi.com", "uid2-token", 1); + entry.ids.insert( + "ids.example.com".to_owned(), + crate::ec::kv_types::KvPartnerId::single("other-uid"), + ); + entry.consent.ok = true; + let snapshot = EcKvSnapshot::Present { + ec_id: ec_id.clone(), + entry: Box::new(entry), + generation: None, + }; + let ec_context = EcContext::new_for_test( + Some(ec_id), + crate::test_support::tests::consent_granting_vendors(vec![1]), + ); + + // Act + let eids = resolve_auction_eids(&snapshot, Some(®istry), &ec_context) + .expect("should resolve EIDs under granted consent"); + + // Assert + let sources: Vec<&str> = eids.iter().map(|eid| eid.source.as_str()).collect(); + assert_eq!( + sources, + vec!["ids.example.com"], + "should not send the GDPR-excluded source to bidders" + ); + } + #[test] fn resolve_auction_eids_returns_none_without_registry() { let ec_id = format!("{}.ABC123", "a".repeat(64)); diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 1dddef193..30c0ef57b 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -280,30 +280,12 @@ pub fn validate_settings_for_deploy(settings: &Settings) -> Result<(), Report Result<(), Report> { - match ec.identity_providers.iter().find(|config| config.enabled) { - Some(config) => Err(Report::new(TrustedServerError::Configuration { - message: format!( - "ec.identity_providers: provider `{}` is enabled, but this release registers no \ - provider kinds; set `enabled = false`", - config.id - ), - })), - None => Ok(()), - } -} - /// Runs Trusted Server runtime validation after secret references are resolved. /// /// # Errors @@ -351,7 +333,8 @@ fn validate_gpt_diagnostics_config(settings: &Settings) -> Result<(), Report Ec { +#[must_use] +pub fn with_placeholder_partner_tokens(ec: &Ec) -> Ec { let mut ec = ec.clone(); for (index, partner) in ec.partners.iter_mut().enumerate() { if partner.api_token.is_some() { @@ -1395,16 +1378,11 @@ gam_network_id = "99999" } #[test] - fn deploy_validation_rejects_an_enabled_provider_while_no_kind_is_registered() { + fn deploy_validation_accepts_an_enabled_provider() { let settings = settings_with_identity_provider(serde_json::json!({})); - let err = validate_settings_for_deploy(&settings) - .expect_err("should reject an enabled provider at deploy time"); - - assert!( - err.to_string().contains("registers no provider kinds"), - "should explain why the provider is rejected: {err:?}" - ); + validate_settings_for_deploy(&settings) + .expect("should leave the registered-kind check to the provider crate"); } #[test] diff --git a/crates/trusted-server-core/src/ec/after_response.rs b/crates/trusted-server-core/src/ec/after_response.rs index b46d3ea48..5e5e0f963 100644 --- a/crates/trusted-server-core/src/ec/after_response.rs +++ b/crates/trusted-server-core/src/ec/after_response.rs @@ -134,7 +134,9 @@ pub fn commit_after_response_lookups( #[cfg(test)] mod tests { use super::*; - use crate::ec::coordinator::results::{DropCounts, LookupStatus, ProviderLookup, SkipReason}; + use crate::ec::coordinator::results::{ + DropCounts, LookupStatus, OptOutMemory, ProviderLookup, SkipReason, + }; use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, SourceDomain, UidValue}; use crate::ec::kv::TombstoneOutcome; use crate::ec::kv_types::{KvEid, KvEntry, KvPartnerId, KvUid}; @@ -190,6 +192,7 @@ mod tests { vec![KvEid::new(EidFields::default(), vec![uid])], )], cleared: Vec::new(), + opt_out: OptOutMemory::Unchanged, }], ) } @@ -507,6 +510,7 @@ mod tests { status: LookupStatus::Skipped(SkipReason::NotDue), entries: Vec::new(), cleared: Vec::new(), + opt_out: OptOutMemory::Unchanged, }], ); @@ -578,6 +582,7 @@ mod tests { status: LookupStatus::Skipped(SkipReason::OptedOut), entries: Vec::new(), cleared, + opt_out: OptOutMemory::Unchanged, }], ) } diff --git a/crates/trusted-server-core/src/ec/coordinator/mod.rs b/crates/trusted-server-core/src/ec/coordinator/mod.rs index c0cd83359..f930532d1 100644 --- a/crates/trusted-server-core/src/ec/coordinator/mod.rs +++ b/crates/trusted-server-core/src/ec/coordinator/mod.rs @@ -15,22 +15,23 @@ use std::time::Duration; use error_stack::Report; -use self::results::{LookupResults, LookupStatus, ProviderLookup, SkipReason}; +use self::results::{LookupResults, LookupStatus, OptOutMemory, ProviderLookup, SkipReason}; use self::transport::{ Deadline, LookupTransport, ProviderIndex, Tagged, TransportError, TransportResponse, }; use self::validate::{ValidatedEntries, ValidationPolicy, validate}; use crate::consent::ConsentContext; +use crate::consent::jurisdiction::Jurisdiction; use crate::consent::{allows_eid_persistence, tcf_vendor_consent_denied}; use crate::ec::EcKvSnapshot; use crate::ec::consent::ec_consent_granted; -use crate::ec::eid_types::{ProviderId, SourceDomain, UnixSeconds}; +use crate::ec::eid_types::{EidFields, Expiry, ProviderId, SourceDomain, UnixSeconds}; use crate::ec::generation::ec_hash; -use crate::ec::kv_types::KvEntry; +use crate::ec::kv_types::{KvEid, KvEntry, KvUid}; use crate::ec::provider::{ - ClientSignals, IdentityProvider, LookupInput, LookupOutcome, OwnedEntries, ProviderConfigError, - ProviderEid, ProviderResponse, SourceInput, destination_allowed, parse_sources, - validate_enabled_configs, + ClientSignals, CookieRead, IdentityProvider, LookupInput, LookupOutcome, OwnedEntries, + ProviderConfigError, ProviderEid, ProviderResponse, RequestCookies, SourceInput, + destination_allowed, parse_sources, validate_enabled_configs, }; use crate::ec::rate_limiter::RateLimiter; use crate::ec::registry::PartnerRegistry; @@ -40,9 +41,9 @@ use crate::settings::IdentityProviderConfig; /// Coordinates server-side identity lookups for one navigation. /// /// Implementations own eligibility, consent, destination checks, the shared -/// time budget and validation. Providers never see KV, cookies, HTTP or the -/// clock. Implementations never write KV: they return the change and the -/// caller spends the [`crate::ec::page_load::PageLoadWrite`] permit. +/// time budget and validation. Providers never see KV, HTTP or the clock, +/// and see cookies only as [`RequestCookies`]. Implementations never write +/// KV: they return the change and the caller spends the [`crate::ec::page_load::PageLoadWrite`] permit. #[allow(async_fn_in_trait)] pub trait IdentityCoordinator { /// Starts lookups for every eligible provider without waiting. @@ -78,6 +79,8 @@ pub struct LookupContext<'a> { pub client: ClientSignals<'a>, /// Publisher domain. pub publisher_domain: &'a str, + /// Request cookies, read by providers that use first-party cookies. + pub cookies: RequestCookies<'a>, /// Existing EC rate limiter, used only for `rate_limit_per_hour`. pub rate_limiter: &'a dyn RateLimiter, /// Request time. @@ -100,8 +103,12 @@ pub struct PendingLookups { early: Vec>, /// Providers whose IDs were generated at dispatch, validated. generated: Vec>, + /// Sources each provider may not collect or keep where GDPR applies. + excluded: Vec>>, /// Provider state to hand back with each in-flight request's response. contexts: Vec>, + /// Providers whose IDs were read from first-party cookies, validated. + from_cookies: Vec>, } impl Default for PendingLookups { @@ -113,7 +120,9 @@ impl Default for PendingLookups { now: UnixSeconds::new(0), early: Vec::new(), generated: Vec::new(), + excluded: Vec::new(), contexts: Vec::new(), + from_cookies: Vec::new(), } } } @@ -124,6 +133,7 @@ impl std::fmt::Debug for PendingLookups { .field("requests", &self.requests.len()) .field("early", &self.early.len()) .field("generated", &self.generated.len()) + .field("from_cookies", &self.from_cookies.len()) .finish_non_exhaustive() } } @@ -132,7 +142,10 @@ impl PendingLookups { /// Whether nothing was dispatched, generated or skipped. #[must_use] pub fn is_empty(&self) -> bool { - self.requests.is_empty() && self.early.is_empty() && self.generated.is_empty() + self.requests.is_empty() + && self.early.is_empty() + && self.generated.is_empty() + && self.from_cookies.is_empty() } } @@ -259,6 +272,20 @@ impl ProviderCoordinator

{ self.providers.is_empty() } + /// Whether `source` holds a UID of this provider whose refresh is due. + fn refresh_due( + registered: &Registered

, + entry: Option<&KvEntry>, + source: &SourceDomain, + now: UnixSeconds, + ) -> bool { + entry + .and_then(|entry| entry.ids.get(source.as_str())) + .is_some_and(|record| { + record.is_provider_written() && record.refresh_due_for(®istered.id, now) + }) + } + /// Whether the row holds a live provider-written UID for `source`. fn holds_live_uid(entry: Option<&KvEntry>, source: &str, now: UnixSeconds) -> bool { entry @@ -304,6 +331,23 @@ impl ProviderCoordinator

{ (due, input_blocked) } + /// The provider's sources it may not collect or keep for this request: + /// those it excludes where GDPR applies, when the visitor's jurisdiction + /// is GDPR. A TCF string alone does not decide this, because some CMPs + /// set one for every visitor. + fn excluded_sources(registered: &Registered

, consent: &ConsentContext) -> Vec { + if !matches!(consent.jurisdiction, Jurisdiction::Gdpr) { + return Vec::new(); + } + registered + .policy + .sources + .iter() + .filter(|source| registered.provider.excluded_where_gdpr_applies(source)) + .cloned() + .collect() + } + /// Whether the provider's hourly limit is reached. Limiter errors fail /// open, matching pull sync. /// @@ -334,7 +378,10 @@ impl ProviderCoordinator

{ entries: ValidatedEntries, ) -> ProviderLookup { let registered = &self.providers[provider.get()]; - let cleared = if status == LookupStatus::Skipped(SkipReason::OptedOut) { + let cleared = if matches!( + status, + LookupStatus::Skipped(SkipReason::OptedOut | SkipReason::Region) + ) { registered.policy.sources.to_vec() } else { Vec::new() @@ -344,6 +391,7 @@ impl ProviderCoordinator

{ status, entries, cleared, + opt_out: OptOutMemory::Unchanged, } } @@ -371,10 +419,9 @@ impl ProviderCoordinator

{ return self.lookup(provider, status, Vec::new()); } }; - if matches!( - response.status, - http::StatusCode::NOT_FOUND | http::StatusCode::NO_CONTENT - ) { + // Only 404 is a match-free reply for every provider; a 204 reaches + // the provider, because some vendors use it to report an opt-out. + if response.status == http::StatusCode::NOT_FOUND { return self.lookup(provider, LookupStatus::NoMatch, Vec::new()); } if !response.status.is_success() { @@ -396,11 +443,17 @@ impl ProviderCoordinator

{ return self.lookup(provider, LookupStatus::NoMatch, Vec::new()); } Ok(LookupOutcome::OptedOut) => { - return self.lookup( - provider, - LookupStatus::Skipped(SkipReason::OptedOut), - Vec::new(), - ); + // Only an opt-out the vendor's server reports is remembered: + // it spares the vendor further calls. Cookie opt-outs are + // re-read on every navigation instead. + return ProviderLookup { + opt_out: OptOutMemory::Remember, + ..self.lookup( + provider, + LookupStatus::Skipped(SkipReason::OptedOut), + Vec::new(), + ) + }; } Err(report) => { log::debug!( @@ -491,10 +544,108 @@ impl IdentityCoordinator for ProviderCoordinator

{ pending.early.push(skip(SkipReason::Consent)); continue; } + let excluded = Self::excluded_sources(registered, lookup.consent); + if !excluded.is_empty() { + if excluded.len() == registered.policy.sources.len() { + pending.early.push(skip(SkipReason::Region)); + continue; + } + pending.excluded.push(Tagged { + provider, + value: excluded.clone(), + }); + } + let remembered_opt_out = + entry.filter(|entry| entry.opted_out(®istered.id, lookup.now)); + let mut opted_back_in = false; + let (mut due, input_blocked) = Self::due_sources(registered, entry, lookup.now); + due.retain(|source| !excluded.contains(source)); + // The vendor's own first-party cookies are read on every + // navigation, so an opt-out or a changed ID is noticed while the + // sources still hold live UIDs. Their IDs stand in for a lookup. + match registered.provider.read_request_cookies( + lookup + .cookies + .at(lookup.now) + .where_gdpr_applies(lookup.consent.gdpr_applies), + ) { + CookieRead::OptedOut => { + log::debug!( + "Identity lookup: provider={} opted out in its cookies", + registered.id + ); + pending.early.push(skip(SkipReason::OptedOut)); + continue; + } + CookieRead::Found(eids) => { + let mut found = self.validated(provider, eids, lookup.now); + // A cookie older than a server-side refresh does not + // roll the refreshed UID back. + if registered.provider.keeps_fresher_stored_uids() { + found.entries.retain(|(source, eids)| { + !stored_outlives_cookie(entry, source, eids, lookup.now) + }); + recount_found(&mut found); + } + // A first-party ID kept for the row's lifetime is not + // replaced by a different cookie value. + if registered.provider.keeps_stored_uids() { + found.entries.retain(|(source, _)| { + !Self::holds_live_uid(entry, source.as_str(), lookup.now) + }); + recount_found(&mut found); + } + if matches!(found.status, LookupStatus::Found { .. }) { + // An ID in the vendor's cookies means the visitor + // opted back in, so a remembered opt-out is + // forgotten. + if remembered_opt_out.is_some() { + opted_back_in = true; + found.opt_out = OptOutMemory::Forget; + } + // Only the sources the cookies filled are spared a + // lookup; the provider's other due sources are + // still looked up. + // A refresh-due source whose cookie only repeats the + // held UID stays due, so its refresh still runs before + // the UID expires. + due.retain(|source| { + found + .entries + .iter() + .find(|(filled, _)| filled == source) + .is_none_or(|(_, eids)| { + Self::refresh_due(registered, entry, source, lookup.now) + && holds_same_uids_for(entry, source, eids, lookup.now) + }) + }); + // A source whose cookie repeats the held UIDs is not + // rewritten: its relative expiry would cause a write + // on every navigation, and a failed lookup for a + // still-due source would replace the held record and + // its refresh state. + found.entries.retain(|(source, eids)| { + !holds_same_uids_for(entry, source, eids, lookup.now) + }); + recount_found(&mut found); + if !found.entries.is_empty() || opted_back_in { + pending.from_cookies.push(Tagged { + provider, + value: found, + }); + if due.is_empty() { + continue; + } + } + } + } + CookieRead::Absent => {} + } // A remembered opt-out keeps the provider from being called until - // it lapses. It clears only sources the row still stores, so once - // they are gone the after-response path skips its re-read. - if let Some(entry) = entry.filter(|entry| entry.opted_out(®istered.id, lookup.now)) { + // it lapses, unless its cookies hold an ID again. It clears only + // sources the row still stores, so once they are gone the + // after-response path skips its re-read. + if let Some(entry) = remembered_opt_out.filter(|_| !opted_back_in) { pending.generated.push(Tagged { provider, value: ProviderLookup { @@ -508,11 +659,11 @@ impl IdentityCoordinator for ProviderCoordinator

{ .filter(|source| entry.ids.contains_key(source.as_str())) .cloned() .collect(), + opt_out: OptOutMemory::Unchanged, }, }); continue; } - let (due, input_blocked) = Self::due_sources(registered, entry, lookup.now); if due.is_empty() { pending.early.push(skip(if input_blocked { SkipReason::Input @@ -528,6 +679,11 @@ impl IdentityCoordinator for ProviderCoordinator

{ publisher_domain: lookup.publisher_domain, due_sources: &due, current: OwnedEntries::new(entry, ®istered.policy.sources, ®istered.id), + cookies: lookup + .cookies + .at(lookup.now) + .where_gdpr_applies(lookup.consent.gdpr_applies), + now: lookup.now, hashed_email: None, }; if let Some(mut eids) = registered.provider.generate(&input) { @@ -547,10 +703,8 @@ impl IdentityCoordinator for ProviderCoordinator

{ continue; } } - if Self::rate_limited(registered, lookup) { - pending.early.push(skip(SkipReason::RateLimited)); - continue; - } + // The request is built before the rate limit is checked, so a + // provider that makes no lookup spends none of its budget. let mut request = match registered.provider.build_request(&input) { Ok(request) => request, Err(report) => { @@ -566,6 +720,10 @@ impl IdentityCoordinator for ProviderCoordinator

{ continue; } }; + if Self::rate_limited(registered, lookup) { + pending.early.push(skip(SkipReason::RateLimited)); + continue; + } if !destination_allowed(&request.url, ®istered.policy.allowed_domains) { pending.early.push(skip(SkipReason::Destination)); continue; @@ -606,7 +764,9 @@ impl IdentityCoordinator for ProviderCoordinator

{ now, early, generated, + excluded, contexts, + from_cookies, } = pending; let responses = match deadline { Some(deadline) if !requests.is_empty() => transport.gather(requests, deadline).await, @@ -617,15 +777,38 @@ impl IdentityCoordinator for ProviderCoordinator

{ for Tagged { provider, value } in early { slots[provider.get()] = Some(self.lookup(provider, value, Vec::new())); } - for Tagged { provider, value } in generated { - slots[provider.get()] = Some(value); + // A provider's cookie result replaces its early skip, and its later + // lookup or generated result is merged into the cookie result. + let mut filled: Vec> = + (0..self.providers.len()).map(|_| None).collect(); + for Tagged { provider, value } in from_cookies.into_iter().chain(generated) { + let slot = &mut filled[provider.get()]; + *slot = Some(match slot.take() { + Some(earlier) => merge_lookups(earlier, value), + None => value, + }); } for Tagged { provider, value } in responses { let context = contexts .iter() .find(|tagged| tagged.provider == provider) .map(|tagged| tagged.value.as_str()); - slots[provider.get()] = Some(self.interpret(provider, value, context, now)); + let later = self.interpret(provider, value, context, now); + let slot = &mut filled[provider.get()]; + *slot = Some(match slot.take() { + Some(earlier) => merge_lookups(earlier, later), + None => later, + }); + } + for (slot, value) in slots.iter_mut().zip(filled) { + if value.is_some() { + *slot = value; + } + } + for Tagged { provider, value } in excluded { + if let Some(lookup) = slots[provider.get()].as_mut() { + exclude_sources(lookup, value); + } } let lookups: Vec = slots.into_iter().flatten().collect(); for lookup in &lookups { @@ -635,11 +818,176 @@ impl IdentityCoordinator for ProviderCoordinator

{ } } +/// Drops `sources` from a provider's result and removes them from the row. +fn exclude_sources(lookup: &mut ProviderLookup, sources: Vec) { + lookup + .entries + .retain(|(source, _)| !sources.contains(source)); + recount_found(lookup); + for source in sources { + if !lookup.cleared.contains(&source) { + lookup.cleared.push(source); + } + } +} + +/// Merges a provider's cookie result with its later lookup or generated +/// result for the sources the cookies did not fill. +/// +/// An opt-out wins; otherwise the found entries of both are kept, and a +/// later result that found nothing leaves the cookie result. +fn merge_lookups(earlier: ProviderLookup, later: ProviderLookup) -> ProviderLookup { + // A server opt-out is remembered over anything; otherwise the cookies' + // opt-back-in is kept. + let opt_out = match (earlier.opt_out, later.opt_out) { + (_, OptOutMemory::Remember) => OptOutMemory::Remember, + (OptOutMemory::Forget, _) => OptOutMemory::Forget, + (_, memory) => memory, + }; + ProviderLookup { + opt_out, + ..merge_entries(earlier, later) + } +} + +/// Merges the statuses and entries of [`merge_lookups`]. +fn merge_entries(earlier: ProviderLookup, later: ProviderLookup) -> ProviderLookup { + let LookupStatus::Found { dropped, .. } = later.status else { + return if later.status == LookupStatus::Skipped(SkipReason::OptedOut) + || !matches!(earlier.status, LookupStatus::Found { .. }) + { + later + } else { + earlier + }; + }; + if !matches!(earlier.status, LookupStatus::Found { .. }) { + return later; + } + let mut entries = later.entries; + for (source, eids) in earlier.entries { + if entries.iter().all(|(held, _)| *held != source) { + entries.push((source, eids)); + } + } + let merged = entries + .iter() + .flat_map(|(_, eids)| eids) + .map(|eid| eid.uids().len()) + .sum::(); + ProviderLookup { + status: LookupStatus::Found { + kept: u16::try_from(merged).unwrap_or(u16::MAX), + dropped, + }, + entries, + ..later + } +} + +/// Recounts the UIDs a found result keeps after some of its entries were +/// dropped; a result left with none becomes `NoMatch`. +fn recount_found(lookup: &mut ProviderLookup) { + if let LookupStatus::Found { kept, .. } = &mut lookup.status { + let remaining = lookup + .entries + .iter() + .flat_map(|(_, eids)| eids) + .map(|eid| eid.uids().len()) + .sum::(); + *kept = u16::try_from(remaining).unwrap_or(u16::MAX); + if remaining == 0 { + lookup.status = LookupStatus::NoMatch; + } + } +} + +/// Whether the row holds a live UID for `source` whose absolute expiry is +/// later than that of every UID in `eids`, so the cookie is older than the +/// stored record. +fn stored_outlives_cookie( + entry: Option<&KvEntry>, + source: &SourceDomain, + eids: &[KvEid], + now: UnixSeconds, +) -> bool { + let Some(record) = entry.and_then(|entry| entry.ids.get(source.as_str())) else { + return false; + }; + let Some(stored_until) = record + .eids() + .iter() + .flat_map(KvEid::uids) + .filter(|uid| uid.is_live(now)) + .filter_map(|uid| match uid.expiry() { + Expiry::At(at) => Some(at), + Expiry::Never => None, + }) + .max() + else { + return false; + }; + eids.iter() + .flat_map(KvEid::uids) + .all(|uid| match uid.expiry() { + Expiry::At(at) => at < stored_until, + Expiry::Never => false, + }) +} + +/// Whether the row holds exactly the live UIDs in `eids` for `source`. +/// +/// UIDs are compared by ID, agent type, `ext` and their EID-level fields; +/// only expiry and refresh state are ignored, so a cookie that changes any +/// stored field is still written. +fn holds_same_uids_for( + entry: Option<&KvEntry>, + source: &SourceDomain, + eids: &[KvEid], + now: UnixSeconds, +) -> bool { + let Some(record) = entry.and_then(|entry| entry.ids.get(source.as_str())) else { + return false; + }; + let held: Vec<(&EidFields, &KvUid)> = record + .eids() + .iter() + .flat_map(|eid| eid.uids().iter().map(move |uid| (eid.fields(), uid))) + .filter(|(_, uid)| uid.is_live(now)) + .collect(); + let read: Vec<(&EidFields, &KvUid)> = eids + .iter() + .flat_map(|eid| eid.uids().iter().map(move |uid| (eid.fields(), uid))) + .collect(); + let same = |(left_fields, left): &(&EidFields, &KvUid), + (right_fields, right): &(&EidFields, &KvUid)| { + left_fields == right_fields + && left.id() == right.id() + && left.atype() == right.atype() + && left.ext() == right.ext() + }; + held.len() == read.len() + && read + .iter() + .all(|read_uid| held.iter().any(|held_uid| same(held_uid, read_uid))) + && held + .iter() + .all(|held_uid| read.iter().any(|read_uid| same(held_uid, read_uid))) +} + fn log_outcome(lookup: &ProviderLookup) { let (kept, dropped) = match lookup.status { LookupStatus::Found { kept, dropped } => (kept, dropped.total()), _ => (0, 0), }; + if let LookupStatus::HttpError(status) = lookup.status { + log::info!( + "Identity lookup: provider={} outcome={} status={status}", + lookup.provider, + lookup.status.metric_name() + ); + return; + } log::info!( "Identity lookup: provider={} outcome={} kept={kept} dropped={dropped}", lookup.provider, @@ -650,9 +998,9 @@ fn log_outcome(lookup: &ProviderLookup) { #[cfg(test)] mod tests { use super::*; - use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::TcfConsent; - use crate::ec::eid_types::{AgentType, EidFields, Expiry, UidValue}; + use crate::ec::coordinator::results::DropCounts; + use crate::ec::eid_types::{AgentType, UidValue}; use crate::ec::kv::KvIdentityGraph; use crate::ec::kv_types::{KvEid, KvPartnerId, KvRefresh, KvUid}; use crate::ec::provider::{ @@ -662,7 +1010,7 @@ mod tests { use crate::error::TrustedServerError; use crate::settings::EcPartner; use std::borrow::Cow; - use std::cell::RefCell; + use std::cell::{Cell, RefCell}; use std::num::NonZeroUsize; const EC_ID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ABC123"; @@ -672,7 +1020,9 @@ mod tests { /// Context the echo provider sets on its request and expects back. const ECHO_CONTEXT: &str = "echo-context"; - struct EchoProvider(ProviderCapabilities); + /// Echoes the response body as one UID; when the cookie name is set, also + /// reads that cookie as one UID. + struct EchoProvider(ProviderCapabilities, Option<&'static str>); impl IdentityProvider for EchoProvider { fn capabilities(&self) -> &ProviderCapabilities { @@ -703,6 +1053,10 @@ mod tests { if response.context != Some(ECHO_CONTEXT) { return Err(Report::new(ProviderError::InvalidResponse)); } + // Like LiveRamp, the echo vendor reports an opt-out with a 204. + if response.status == http::StatusCode::NO_CONTENT { + return Ok(LookupOutcome::OptedOut); + } let id = std::str::from_utf8(response.body) .map_err(|_| Report::new(ProviderError::InvalidResponse))?; if id == "optout" { @@ -723,6 +1077,32 @@ mod tests { }], }])) } + + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> CookieRead<'cookie> { + let Some(id) = self.1.and_then(|name| cookies.get(name)) else { + return CookieRead::Absent; + }; + if id == "optout" { + return CookieRead::OptedOut; + } + CookieRead::Found(vec![ProviderEid { + source: Cow::Borrowed("ids.example.com"), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids: vec![ProviderUid { + id: Cow::Borrowed(id), + atype: None, + ext: None, + expiry: ProviderExpiry::At(UnixSeconds::new(5000)), + refresh: None, + }], + }]) + } } /// Answers every request with a canned result, recording the URLs sent. @@ -823,22 +1203,29 @@ mod tests { } fn echo_provider(input: SourceInput) -> EchoProvider { - EchoProvider(ProviderCapabilities { - sources: Box::new([SourceCapability { - source: SourceDomain::try_from("ids.example.com").expect("should parse"), - input, - }]), - shape: EidShape { - max_eids: NonZeroUsize::new(4).expect("should be non-zero"), - max_uids_per_eid: NonZeroUsize::new(4).expect("should be non-zero"), - eid_level_fields: EidLevelFields::Preserved, + cookie_provider(input, None) + } + + fn cookie_provider(input: SourceInput, cookie: Option<&'static str>) -> EchoProvider { + EchoProvider( + ProviderCapabilities { + sources: Box::new([SourceCapability { + source: SourceDomain::try_from("ids.example.com").expect("should parse"), + input, + }]), + shape: EidShape { + max_eids: NonZeroUsize::new(4).expect("should be non-zero"), + max_uids_per_eid: NonZeroUsize::new(4).expect("should be non-zero"), + eid_level_fields: EidLevelFields::Preserved, + }, }, - }) + cookie, + ) } - fn coordinator_with( - registered: RegisteredProvider, - ) -> ProviderCoordinator { + fn coordinator_with( + registered: RegisteredProvider

, + ) -> ProviderCoordinator

{ ProviderCoordinator::new(vec![registered], partners(), Duration::from_millis(200)) .expect("should build coordinator") } @@ -880,6 +1267,24 @@ mod tests { snapshot: &EcKvSnapshot, consent: &ConsentContext, limiter: &dyn RateLimiter, + ) -> LookupResults { + run_with_cookies( + coordinator, + transport, + snapshot, + consent, + limiter, + "vendor_id=c-1", + ) + } + + fn run_with_cookies( + coordinator: &ProviderCoordinator

, + transport: &CannedTransport, + snapshot: &EcKvSnapshot, + consent: &ConsentContext, + limiter: &dyn RateLimiter, + cookies: &str, ) -> LookupResults { let context = LookupContext { ec_id: EC_ID, @@ -890,6 +1295,7 @@ mod tests { user_agent: None, }, publisher_domain: "www.example.com", + cookies: RequestCookies::new(cookies), rate_limiter: limiter, now: UnixSeconds::new(1000), }; @@ -969,7 +1375,7 @@ mod tests { fn provider_record(id: &str) -> KvPartnerId { let uid = KvUid::new( UidValue::try_from(id).expect("should build uid"), - AgentType::try_from(1).expect("should build atype"), + AgentType::try_from(3).expect("should build atype"), None, Some(ProviderId::new("example-provider")), Expiry::Never, @@ -1193,6 +1599,101 @@ mod tests { ); } + /// An echo provider whose IDs are not offered where GDPR applies. + struct OutsideGdprProvider(EchoProvider); + + impl IdentityProvider for OutsideGdprProvider { + fn capabilities(&self) -> &ProviderCapabilities { + self.0.capabilities() + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + self.0.build_request(lookup) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + self.0.parse_response(response) + } + + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> CookieRead<'cookie> { + self.0.read_request_cookies(cookies) + } + + fn excluded_where_gdpr_applies(&self, _source: &SourceDomain) -> bool { + true + } + } + + #[test] + fn a_provider_outside_gdpr_regions_is_skipped_where_gdpr_applies() { + let coordinator = ProviderCoordinator::new( + vec![RegisteredProvider::new( + provider_config(""), + OutsideGdprProvider(cookie_provider(SourceInput::Anonymous, Some("vendor_id"))), + )], + partners(), + Duration::from_millis(200), + ) + .expect("should build coordinator"); + let transport = CannedTransport::new(ok_reply); + + let gdpr = run(&coordinator, &transport, &live_row(), &tcf_consent(vec![])); + let outside = run(&coordinator, &transport, &live_row(), &granting()); + + let lookup = gdpr.iter().next().expect("should report the provider"); + assert_eq!( + lookup.status, + LookupStatus::Skipped(SkipReason::Region), + "should neither read cookies nor look up where GDPR applies" + ); + assert_eq!( + lookup.cleared, + [SourceDomain::try_from("ids.example.com").expect("should parse")], + "should remove the stored UIDs where GDPR applies" + ); + assert!( + matches!(only_status(&outside), LookupStatus::Found { .. }), + "should read the cookie where GDPR does not apply" + ); + } + + #[test] + fn a_tcf_string_alone_does_not_exclude_a_provider_outside_gdpr() { + // Arrange: a CMP that sets a TCF string for a US visitor. + let coordinator = ProviderCoordinator::new( + vec![RegisteredProvider::new( + provider_config(""), + OutsideGdprProvider(cookie_provider(SourceInput::Anonymous, Some("vendor_id"))), + )], + partners(), + Duration::from_millis(200), + ) + .expect("should build coordinator"); + let transport = CannedTransport::new(ok_reply); + let consent = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + ..tcf_consent(vec![]) + }; + + // Act + let results = run(&coordinator, &transport, &live_row(), &consent); + + // Assert + assert!( + matches!(only_status(&results), LookupStatus::Found { .. }), + "should collect where only a TCF string is present" + ); + } + #[test] fn hashed_email_sources_are_skipped_for_input() { let coordinator = coordinator_with(RegisteredProvider::new( @@ -1287,48 +1788,189 @@ mod tests { } #[test] - fn disabled_provider_is_not_registered() { - let coordinator = coordinator("enabled = false"); + fn first_party_cookie_id_replaces_the_lookup() { + let coordinator = coordinator_with(RegisteredProvider::new( + provider_config(""), + cookie_provider(SourceInput::Anonymous, Some("vendor_id")), + )); + let transport = CannedTransport::new(ok_reply); - assert!(coordinator.is_empty(), "should drop disabled providers"); + let results = run(&coordinator, &transport, &live_row(), &granting()); + + let lookup = results.iter().next().expect("should report the provider"); + assert!( + matches!(lookup.status, LookupStatus::Found { kept: 1, .. }), + "should keep the cookie UID" + ); + assert_eq!( + lookup.entries[0].1[0].uids()[0].id().as_str(), + "c-1", + "should store the cookie value" + ); + assert!(transport.sent.borrow().is_empty(), "should send no lookup"); + } + + fn cookie_coordinator() -> ProviderCoordinator { + coordinator_with(RegisteredProvider::new( + provider_config(""), + cookie_provider(SourceInput::Anonymous, Some("vendor_id")), + )) + } + + fn held_snapshot(id: &str) -> EcKvSnapshot { + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry + .ids + .insert("ids.example.com".to_owned(), provider_record(id)); + seeded_snapshot(&entry) } #[test] - fn endpoint_outside_allowlist_is_rejected_at_build() { - let mut config = provider_config(""); - config.endpoint = "https://other.example.com/".to_owned(); + fn cookie_opt_out_skips_the_lookup_and_clears_the_sources() { + let coordinator = cookie_coordinator(); + let transport = CannedTransport::new(ok_reply); - let error = ProviderCoordinator::new( - vec![RegisteredProvider::new( - config, - echo_provider(SourceInput::Anonymous), - )], - partners(), - Duration::from_millis(200), - ) - .err() - .expect("should reject endpoint"); + for snapshot in [live_row(), held_snapshot("p-1")] { + let results = run_with_cookies( + &coordinator, + &transport, + &snapshot, + &granting(), + &Limiter(false), + "vendor_id=optout", + ); - assert!( - matches!(error.current_context(), ProviderConfigError::Invalid), - "should report invalid config" - ); + let lookup = results.iter().next().expect("should report the provider"); + assert_eq!( + lookup.status, + LookupStatus::Skipped(SkipReason::OptedOut), + "should skip for the opt-out" + ); + assert_eq!( + lookup.cleared, + [SourceDomain::try_from("ids.example.com").expect("should parse")], + "should clear the provider's sources" + ); + } + assert!(transport.sent.borrow().is_empty(), "should send no lookup"); } #[test] - fn source_outside_capabilities_is_rejected_at_build() { - let provider = EchoProvider(ProviderCapabilities { - sources: Box::new([]), - ..echo_provider(SourceInput::Anonymous).0 - }); + fn changed_cookie_replaces_a_held_live_uid() { + let coordinator = cookie_coordinator(); + let transport = CannedTransport::new(ok_reply); - let error = ProviderCoordinator::new( - vec![RegisteredProvider::new(provider_config(""), provider)], - partners(), - Duration::from_millis(200), - ) - .err() - .expect("should reject source outside capabilities"); + let results = run(&coordinator, &transport, &held_snapshot("p-1"), &granting()); + + let lookup = results.iter().next().expect("should report the provider"); + assert!( + matches!(lookup.status, LookupStatus::Found { kept: 1, .. }), + "should read the changed cookie although the source is not due" + ); + assert_eq!( + lookup.entries[0].1[0].uids()[0].id().as_str(), + "c-1", + "should store the new cookie value" + ); + assert!(transport.sent.borrow().is_empty(), "should send no lookup"); + } + + #[test] + fn unchanged_cookie_for_a_held_uid_is_not_rewritten() { + let coordinator = cookie_coordinator(); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &held_snapshot("c-1"), &granting()); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::NotDue), + "should skip a cookie repeating the held UID" + ); + assert!(transport.sent.borrow().is_empty(), "should send no lookup"); + } + + #[test] + fn missing_or_invalid_cookie_falls_back_to_the_lookup() { + let coordinator = coordinator_with(RegisteredProvider::new( + provider_config(""), + cookie_provider(SourceInput::Anonymous, Some("absent")), + )); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &live_row(), &granting()); + + assert!( + matches!(only_status(&results), LookupStatus::Found { kept: 1, .. }), + "should use the lookup" + ); + assert_eq!(transport.sent.borrow().len(), 1, "should send the lookup"); + } + + #[test] + fn cookie_is_not_read_without_consent() { + let coordinator = coordinator_with(RegisteredProvider::new( + provider_config("tcf_vendor_id = 9999"), + cookie_provider(SourceInput::Anonymous, Some("vendor_id")), + )); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &live_row(), &tcf_consent(vec![])); + + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::Consent), + "should skip for consent" + ); + } + + #[test] + fn disabled_provider_is_not_registered() { + let coordinator = coordinator("enabled = false"); + + assert!(coordinator.is_empty(), "should drop disabled providers"); + } + + #[test] + fn endpoint_outside_allowlist_is_rejected_at_build() { + let mut config = provider_config(""); + config.endpoint = Some("https://other.example.com/".to_owned()); + + let error = ProviderCoordinator::new( + vec![RegisteredProvider::new( + config, + echo_provider(SourceInput::Anonymous), + )], + partners(), + Duration::from_millis(200), + ) + .err() + .expect("should reject endpoint"); + + assert!( + matches!(error.current_context(), ProviderConfigError::Invalid), + "should report invalid config" + ); + } + + #[test] + fn source_outside_capabilities_is_rejected_at_build() { + let provider = EchoProvider( + ProviderCapabilities { + sources: Box::new([]), + ..echo_provider(SourceInput::Anonymous).0 + }, + None, + ); + + let error = ProviderCoordinator::new( + vec![RegisteredProvider::new(provider_config(""), provider)], + partners(), + Duration::from_millis(200), + ) + .err() + .expect("should reject source outside capabilities"); assert!( matches!(error.current_context(), ProviderConfigError::Invalid), @@ -1343,7 +1985,7 @@ mod tests { fn refreshable_record_owned_by(owner: &str, from: u64) -> KvPartnerId { let uid = KvUid::new( UidValue::try_from("p-1").expect("should build uid"), - AgentType::try_from(1).expect("should build atype"), + AgentType::try_from(3).expect("should build atype"), None, Some(ProviderId::new(owner)), Expiry::At(UnixSeconds::new(5000)), @@ -1385,6 +2027,119 @@ mod tests { assert_eq!(transport.sent.borrow().len(), 1, "should send the refresh"); } + #[test] + fn a_due_refresh_runs_while_the_cookie_repeats_the_held_uid() { + let coordinator = cookie_coordinator(); + let transport = CannedTransport::new(ok_reply); + + run_with_cookies( + &coordinator, + &transport, + &snapshot_holding(refreshable_record(900)), + &granting(), + &Limiter(false), + "vendor_id=p-1", + ); + + assert_eq!( + transport.sent.borrow().len(), + 1, + "should send the refresh although the cookie holds the same UID" + ); + } + + #[test] + fn a_newer_cookie_uid_replaces_a_refresh() { + let coordinator = cookie_coordinator(); + let transport = CannedTransport::new(ok_reply); + + run_with_cookies( + &coordinator, + &transport, + &snapshot_holding(refreshable_record(900)), + &granting(), + &Limiter(false), + "vendor_id=c-2", + ); + + assert!( + transport.sent.borrow().is_empty(), + "should take the newer cookie UID instead of refreshing" + ); + } + + fn found_lookup(source: &str, id: &str) -> ProviderLookup { + let uid = KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::Never, + ); + ProviderLookup { + provider: ProviderId::new("example-provider"), + status: LookupStatus::Found { + kept: 1, + dropped: DropCounts::default(), + }, + entries: vec![( + SourceDomain::try_from(source).expect("should build source"), + vec![KvEid::new(EidFields::default(), vec![uid])], + )], + cleared: Vec::new(), + opt_out: OptOutMemory::Unchanged, + } + } + + fn status_lookup(status: LookupStatus) -> ProviderLookup { + ProviderLookup { + provider: ProviderId::new("example-provider"), + status, + entries: Vec::new(), + cleared: Vec::new(), + opt_out: OptOutMemory::Unchanged, + } + } + + #[test] + fn merging_keeps_cookie_ids_when_the_lookup_fails_and_lets_an_opt_out_win() { + for failed in [ + LookupStatus::Timeout, + LookupStatus::HttpError(http::StatusCode::BAD_GATEWAY), + ] { + let merged = merge_lookups(found_lookup("a.example.com", "c-1"), status_lookup(failed)); + assert!( + matches!(merged.status, LookupStatus::Found { kept: 1, .. }), + "should keep the cookie IDs when the lookup fails" + ); + } + + let merged = merge_lookups( + found_lookup("a.example.com", "c-1"), + status_lookup(LookupStatus::Skipped(SkipReason::OptedOut)), + ); + + assert_eq!( + merged.status, + LookupStatus::Skipped(SkipReason::OptedOut), + "should let a lookup opt-out win over cookie IDs" + ); + } + + #[test] + fn merging_combines_cookie_and_lookup_sources() { + let merged = merge_lookups( + found_lookup("a.example.com", "c-1"), + found_lookup("b.example.com", "l-1"), + ); + + assert!( + matches!(merged.status, LookupStatus::Found { kept: 2, .. }), + "should count both sources" + ); + assert_eq!(merged.entries.len(), 2, "should keep both sources"); + } + #[test] fn a_future_refresh_leaves_the_source_held() { let coordinator = coordinator(""); @@ -1405,6 +2160,80 @@ mod tests { assert!(transport.sent.borrow().is_empty(), "should send nothing"); } + #[test] + fn a_cookie_id_clears_a_remembered_opt_out() { + // Arrange: the opt-out is remembered for 29 more days, and the + // vendor's cookie holds an ID again. + let coordinator = cookie_coordinator(); + let mut entry = KvEntry::tombstone(1000); + entry.consent.ok = true; + entry.opt_outs.insert( + ProviderId::new("example-provider"), + 1000 + 29 * 24 * 60 * 60, + ); + let transport = CannedTransport::new(ok_reply); + + // Act + let results = run_with_cookies( + &coordinator, + &transport, + &seeded_snapshot(&entry), + &granting(), + &Limiter(false), + "vendor_id=c-1", + ); + + // Assert + let lookup = results.iter().next().expect("should report the provider"); + assert!( + matches!(lookup.status, LookupStatus::Found { kept: 1, .. }), + "should store the cookie ID despite the remembered opt-out" + ); + assert_eq!( + lookup.opt_out, + OptOutMemory::Forget, + "should forget the remembered opt-out" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn only_a_vendor_server_opt_out_is_remembered() { + let cookie_only = cookie_coordinator(); + let cookie = run_with_cookies( + &cookie_only, + &CannedTransport::new(ok_reply), + &live_row(), + &granting(), + &Limiter(false), + "vendor_id=optout", + ); + let server = run( + &coordinator(""), + &CannedTransport::new(optout_reply), + &live_row(), + &granting(), + ); + + let memory = |results: &LookupResults| { + results + .iter() + .next() + .expect("should report the provider") + .opt_out + }; + assert_eq!( + memory(&cookie), + OptOutMemory::Unchanged, + "should not remember a cookie opt-out" + ); + assert_eq!( + memory(&server), + OptOutMemory::Remember, + "should remember a vendor server's opt-out" + ); + } + #[test] fn remembered_opt_out_skips_the_provider_until_it_lapses() { // Arrange @@ -1537,6 +2366,44 @@ mod tests { ); } + fn no_content_reply() -> Result> { + Ok(reply(http::StatusCode::NO_CONTENT, b"")) + } + + #[test] + fn a_204_reply_reaches_the_provider() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(no_content_reply); + + let results = run(&coordinator, &transport, &live_row(), &granting()); + + let lookup = results.iter().next().expect("should report the provider"); + assert_eq!( + lookup.status, + LookupStatus::Skipped(SkipReason::OptedOut), + "should report the opt-out the provider reads from a 204" + ); + assert_eq!( + lookup.cleared, + [SourceDomain::try_from("ids.example.com").expect("should parse")], + "should clear the provider's sources" + ); + } + + #[test] + fn a_404_reply_is_no_match_without_parsing() { + let coordinator = coordinator(""); + let transport = CannedTransport::new(not_found_reply); + + let results = run(&coordinator, &transport, &live_row(), &granting()); + + assert_eq!( + only_status(&results), + LookupStatus::NoMatch, + "should report a 404 as no match" + ); + } + /// An echo provider whose request can never be built. struct UnbuildableProvider(EchoProvider); @@ -1721,4 +2588,544 @@ mod tests { "should keep the held ID" ); } + + /// An echo provider that makes no lookup. + struct NoLookupProvider(EchoProvider); + + impl IdentityProvider for NoLookupProvider { + fn capabilities(&self) -> &ProviderCapabilities { + self.0.capabilities() + } + + fn build_request( + &self, + _lookup: &LookupInput<'_>, + ) -> Result> { + Err(Report::new(ProviderError::Request)) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + self.0.parse_response(response) + } + } + + /// A limiter that counts the checks it is asked for. + #[derive(Default)] + struct CountingLimiter(Cell); + + impl RateLimiter for CountingLimiter { + fn exceeded( + &self, + _key: &str, + _hourly_limit: u32, + ) -> Result> { + self.0.set(self.0.get() + 1); + Ok(false) + } + } + + #[test] + fn a_provider_making_no_lookup_spends_no_rate_limit() { + let coordinator = ProviderCoordinator::new( + vec![RegisteredProvider::new( + provider_config("rate_limit_per_hour = 1"), + NoLookupProvider(echo_provider(SourceInput::Anonymous)), + )], + partners(), + Duration::from_millis(200), + ) + .expect("should build coordinator"); + let transport = CannedTransport::new(ok_reply); + let limiter = CountingLimiter::default(); + + let results = run_with(&coordinator, &transport, &live_row(), &granting(), &limiter); + + assert_eq!( + only_status(&results), + LookupStatus::RequestFailed, + "should report a provider that builds no request" + ); + assert_eq!(limiter.0.get(), 0, "should not check the rate limit"); + } + + const SECOND_SOURCE: &str = "ids-b.example.com"; + + /// A two-source provider: its cookie fills the first source and its + /// lookup the second. + struct TwoSourceProvider(EchoProvider); + + impl IdentityProvider for TwoSourceProvider { + fn capabilities(&self) -> &ProviderCapabilities { + self.0.capabilities() + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + self.0.build_request(lookup) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + let mut outcome = self.0.parse_response(response)?; + if let LookupOutcome::Found(eids) = &mut outcome { + for eid in eids { + eid.source = Cow::Borrowed(SECOND_SOURCE); + } + } + Ok(outcome) + } + + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> CookieRead<'cookie> { + self.0.read_request_cookies(cookies) + } + } + + fn two_source_coordinator( + wrap: impl FnOnce(TwoSourceProvider) -> P, + ) -> ProviderCoordinator

{ + let mut provider = cookie_provider(SourceInput::Anonymous, Some("vendor_id")); + provider.0.sources = Box::new([ + SourceCapability { + source: SourceDomain::try_from("ids.example.com").expect("should parse"), + input: SourceInput::Anonymous, + }, + SourceCapability { + source: SourceDomain::try_from(SECOND_SOURCE).expect("should parse"), + input: SourceInput::Anonymous, + }, + ]); + let config: IdentityProviderConfig = toml::from_str(&format!( + "id = \"example-provider\"\nkind = \"echo\"\nendpoint = \"https://ids.example.com/lookup\"\nallowed_domains = [\"ids.example.com\"]\nsources = [\"ids.example.com\", \"{SECOND_SOURCE}\"]\nrate_limit_per_hour = 1\n" + )) + .expect("should parse config"); + let partners: Vec = ["ids.example.com", SECOND_SOURCE] + .into_iter() + .map(|source| { + serde_json::from_value(serde_json::json!({ + "name": source, "source_domain": source, "bidstream_enabled": true + })) + .expect("should build partner") + }) + .collect(); + ProviderCoordinator::new( + vec![RegisteredProvider::new( + config, + wrap(TwoSourceProvider(provider)), + )], + Arc::new(PartnerRegistry::from_config(&partners).expect("should build registry")), + Duration::from_millis(200), + ) + .expect("should build coordinator") + } + + #[test] + fn a_cookie_spares_only_the_sources_it_fills() { + let coordinator = two_source_coordinator(|provider| provider); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &live_row(), &granting()); + + assert_eq!( + *transport.sent.borrow(), + ["https://ids.example.com/lookup?due=1"], + "should look up only the source the cookie did not fill" + ); + let lookup = results.iter().next().expect("should report the provider"); + assert!( + matches!(lookup.status, LookupStatus::Found { kept: 2, .. }), + "should keep the cookie and lookup UIDs" + ); + let mut sources: Vec<&str> = lookup + .entries + .iter() + .map(|(source, _)| source.as_str()) + .collect(); + sources.sort_unstable(); + assert_eq!( + sources, + ["ids-b.example.com", "ids.example.com"], + "should store both sources" + ); + } + + /// Plans the row change a navigation's results make, as finalize does. + fn planned_change( + snapshot: &EcKvSnapshot, + results: &LookupResults, + ) -> Option { + let partners: Vec = ["ids.example.com", SECOND_SOURCE] + .into_iter() + .map(|source| { + serde_json::from_value(serde_json::json!({ + "name": source, "source_domain": source, "bidstream_enabled": true + })) + .expect("should build partner") + }) + .collect(); + let registry = PartnerRegistry::from_config(&partners).expect("should build registry"); + crate::ec::write_plan::plan_write(&crate::ec::write_plan::WritePlanInput { + ec_id: EC_ID, + snapshot, + new_entry: None, + browser_ingest: &[], + lookups: results, + registry: ®istry, + now: UnixSeconds::new(1000), + }) + } + + /// Outcomes of a lookup that stores nothing, with a label for messages. + fn failed_outcomes() -> [(&'static str, Reply, bool); 5] { + [ + ("404", not_found_reply, false), + ("timeout", timeout_reply, false), + ("502", server_error_reply, false), + ("invalid", invalid_reply, false), + ("rate-limited", ok_reply, true), + ] + } + + #[test] + fn an_unchanged_cookie_is_not_rewritten_while_another_source_is_due() { + // Arrange: the row holds the cookie's UID; the second source is due. + let coordinator = two_source_coordinator(|provider| provider); + let snapshot = held_snapshot("c-1"); + + for (label, reply, limited) in failed_outcomes() { + for cookies in ["vendor_id=c-1", ""] { + let transport = CannedTransport::new(reply); + + // Act + let results = run_with_cookies( + &coordinator, + &transport, + &snapshot, + &granting(), + &Limiter(limited), + cookies, + ); + + // Assert + assert!( + planned_change(&snapshot, &results).is_none(), + "should plan no write for {label} with cookies `{cookies}`: {:?}", + only_status(&results) + ); + } + } + } + + #[test] + fn a_failed_refresh_keeps_the_held_refresh_state() { + // Arrange: the held UID's refresh is due and the cookie repeats it. + let coordinator = coordinator_with(RegisteredProvider::new( + provider_config("rate_limit_per_hour = 1"), + cookie_provider(SourceInput::Anonymous, Some("vendor_id")), + )); + let snapshot = snapshot_holding(refreshable_record(900)); + + for (label, reply, limited) in failed_outcomes() { + for cookies in ["vendor_id=p-1", ""] { + let transport = CannedTransport::new(reply); + + // Act + let results = run_with_cookies( + &coordinator, + &transport, + &snapshot, + &granting(), + &Limiter(limited), + cookies, + ); + + // Assert + let lookup = results.iter().next().expect("should report the provider"); + assert!( + lookup.entries.is_empty(), + "should not replace the held record after {label}: {:?}", + lookup.status + ); + assert!( + planned_change(&snapshot, &results).is_none(), + "should plan no write that drops the refresh state for {label} with cookies `{cookies}`" + ); + } + } + } + + /// A cookie provider that keeps a stored UID outliving the cookie's. + struct RefreshingCookieProvider(EchoProvider); + + impl IdentityProvider for RefreshingCookieProvider { + fn capabilities(&self) -> &ProviderCapabilities { + self.0.capabilities() + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + self.0.build_request(lookup) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + self.0.parse_response(response) + } + + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> CookieRead<'cookie> { + self.0.read_request_cookies(cookies) + } + + fn keeps_fresher_stored_uids(&self) -> bool { + true + } + } + + fn record_until(id: &str, until: u64) -> KvPartnerId { + let uid = KvUid::new( + UidValue::try_from(id).expect("should build uid"), + AgentType::try_from(3).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::At(UnixSeconds::new(until)), + ); + KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![uid])], + UnixSeconds::new(1000), + ) + .expect("should build provider record") + } + + #[test] + fn an_older_cookie_does_not_roll_back_a_refreshed_uid() { + // Arrange: the cookie's UID expires at 5000; a refresh stored one + // living until 6000. + let coordinator = coordinator_with(RegisteredProvider::new( + provider_config(""), + RefreshingCookieProvider(cookie_provider(SourceInput::Anonymous, Some("vendor_id"))), + )); + let transport = CannedTransport::new(ok_reply); + + // Act + let results = run( + &coordinator, + &transport, + &snapshot_holding(record_until("p-2", 6000)), + &granting(), + ); + + // Assert + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::NotDue), + "should keep the refreshed UID over the older cookie" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + struct StableCookieProvider(EchoProvider); + + impl IdentityProvider for StableCookieProvider { + fn capabilities(&self) -> &ProviderCapabilities { + self.0.capabilities() + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + self.0.build_request(lookup) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + self.0.parse_response(response) + } + + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> CookieRead<'cookie> { + self.0.read_request_cookies(cookies) + } + + fn keeps_stored_uids(&self) -> bool { + true + } + } + + #[test] + fn a_different_cookie_does_not_replace_a_kept_stored_uid() { + // Arrange: the row holds a live UID; the cookie carries another, + // longer-lived one. + let coordinator = coordinator_with(RegisteredProvider::new( + provider_config(""), + StableCookieProvider(cookie_provider(SourceInput::Anonymous, Some("vendor_id"))), + )); + let transport = CannedTransport::new(ok_reply); + + // Act + let results = run( + &coordinator, + &transport, + &snapshot_holding(record_until("p-2", 4000)), + &granting(), + ); + + // Assert + assert_eq!( + only_status(&results), + LookupStatus::Skipped(SkipReason::NotDue), + "should keep the stored UID over a different cookie" + ); + assert!(transport.sent.borrow().is_empty(), "should send nothing"); + } + + #[test] + fn a_kept_source_without_a_stored_uid_takes_the_cookie() { + let coordinator = coordinator_with(RegisteredProvider::new( + provider_config(""), + StableCookieProvider(cookie_provider(SourceInput::Anonymous, Some("vendor_id"))), + )); + let transport = CannedTransport::new(ok_reply); + + let results = run(&coordinator, &transport, &live_row(), &granting()); + + assert!( + matches!(only_status(&results), LookupStatus::Found { kept: 1, .. }), + "should store the cookie UID when none is held" + ); + } + + #[test] + fn a_newer_cookie_replaces_an_older_stored_uid() { + let coordinator = coordinator_with(RegisteredProvider::new( + provider_config(""), + RefreshingCookieProvider(cookie_provider(SourceInput::Anonymous, Some("vendor_id"))), + )); + let transport = CannedTransport::new(ok_reply); + + let results = run( + &coordinator, + &transport, + &snapshot_holding(record_until("p-2", 4000)), + &granting(), + ); + + assert!( + matches!(only_status(&results), LookupStatus::Found { kept: 1, .. }), + "should take the cookie UID that outlives the stored one" + ); + } + + #[test] + fn a_cookie_changing_only_the_agent_type_is_rewritten() { + let coordinator = cookie_coordinator(); + let transport = CannedTransport::new(ok_reply); + let uid = KvUid::new( + UidValue::try_from("c-1").expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("example-provider")), + Expiry::Never, + ); + let record = KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![uid])], + UnixSeconds::new(1000), + ) + .expect("should build provider record"); + + let results = run( + &coordinator, + &transport, + &snapshot_holding(record), + &granting(), + ); + + assert!( + matches!(only_status(&results), LookupStatus::Found { kept: 1, .. }), + "should store a cookie UID whose agent type changed" + ); + } + + /// A two-source provider whose cookie source is excluded where GDPR + /// applies. + struct CookieSourceExcluded(TwoSourceProvider); + + impl IdentityProvider for CookieSourceExcluded { + fn capabilities(&self) -> &ProviderCapabilities { + self.0.capabilities() + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + self.0.build_request(lookup) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + self.0.parse_response(response) + } + + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> CookieRead<'cookie> { + self.0.read_request_cookies(cookies) + } + + fn excluded_where_gdpr_applies(&self, source: &SourceDomain) -> bool { + source.as_str() == "ids.example.com" + } + } + + #[test] + fn an_excluded_source_is_neither_collected_nor_kept_where_gdpr_applies() { + // Arrange + let coordinator = two_source_coordinator(CookieSourceExcluded); + let transport = CannedTransport::new(ok_reply); + + // Act + let results = run(&coordinator, &transport, &live_row(), &tcf_consent(vec![])); + + // Assert + let lookup = results.iter().next().expect("should report the provider"); + let sources: Vec<&str> = lookup + .entries + .iter() + .map(|(source, _)| source.as_str()) + .collect(); + assert_eq!( + sources, + [SECOND_SOURCE], + "should keep only the source that is not excluded" + ); + assert_eq!( + lookup.cleared, + [SourceDomain::try_from("ids.example.com").expect("should parse")], + "should remove the excluded source from the row" + ); + } } diff --git a/crates/trusted-server-core/src/ec/coordinator/results.rs b/crates/trusted-server-core/src/ec/coordinator/results.rs index f5015e452..43a86103f 100644 --- a/crates/trusted-server-core/src/ec/coordinator/results.rs +++ b/crates/trusted-server-core/src/ec/coordinator/results.rs @@ -63,9 +63,12 @@ pub enum SkipReason { Destination, /// The provider's hourly rate limit is reached. RateLimited, - /// The vendor reports an opt-out; the provider's stored UIDs are - /// removed. + /// The vendor or its cookies report an opt-out; the provider's stored + /// UIDs are removed. OptedOut, + /// Every source of the provider is excluded where GDPR applies; the + /// provider's stored UIDs are removed. + Region, } /// One outcome per provider per navigation. @@ -111,12 +114,27 @@ impl LookupStatus { Self::Skipped(SkipReason::NotDue) => "skipped_not_due", Self::Skipped(SkipReason::Input) => "skipped_input", Self::Skipped(SkipReason::RateLimited) => "skipped_rate_limited", + Self::Skipped(SkipReason::Region) => "skipped_region", Self::Skipped(SkipReason::Destination) => "destination_rejected", Self::Skipped(SkipReason::OptedOut) => "skipped_opted_out", } } } +/// What a lookup does to the provider's remembered opt-out. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum OptOutMemory { + /// Leaves it as it is. + #[default] + Unchanged, + /// Remembers an opt-out the vendor's server reported, so the vendor is + /// not called again for the memory period. + Remember, + /// Forgets it: the vendor's cookies hold an ID again, so the visitor + /// opted back in. + Forget, +} + /// One provider's outcome and validated entries. #[derive(Debug, Clone)] pub struct ProviderLookup { @@ -129,6 +147,8 @@ pub struct ProviderLookup { /// Sources whose stored records are removed, because the user opted /// out of the provider; empty unless `Skipped(OptedOut)`. pub cleared: Vec, + /// What this lookup does to the provider's remembered opt-out. + pub opt_out: OptOutMemory, } /// Validated results of one navigation's lookups. diff --git a/crates/trusted-server-core/src/ec/coordinator/validate.rs b/crates/trusted-server-core/src/ec/coordinator/validate.rs index 9d9a63b17..bd63245b5 100644 --- a/crates/trusted-server-core/src/ec/coordinator/validate.rs +++ b/crates/trusted-server-core/src/ec/coordinator/validate.rs @@ -13,6 +13,9 @@ use crate::ec::registry::PartnerRegistry; /// Largest private refresh state a provider may keep with one UID. const MAX_REFRESH_STATE_BYTES: usize = 2048; +/// Longest a provider-stated expiry, absolute or relative, may lie after the +/// request: 30 days. Cookie- and vendor-supplied times are clamped to it. +const MAX_EXPIRY_SECS: u64 = 30 * 24 * 60 * 60; /// Policy one response is validated against. pub(crate) struct ValidationPolicy<'a> { @@ -117,7 +120,7 @@ fn eid_fields( inserter: bounded_field(eid.inserter.as_ref())?, matcher: bounded_field(eid.matcher.as_ref())?, mm: eid.mm.map(MatchMethod::try_from).transpose()?, - ext: eid.ext.map(RawJson::try_from).transpose()?, + ext: eid.ext.as_deref().map(RawJson::try_from).transpose()?, }) } @@ -128,7 +131,7 @@ fn stored_uid( ) -> Result { let id = UidValue::try_from(uid.id.into_owned())?; let atype = AgentType::try_from(uid.atype.unwrap_or(default_atype))?; - let ext = uid.ext.map(RawJson::try_from).transpose()?; + let ext = uid.ext.as_deref().map(RawJson::try_from).transpose()?; let expiry = match resolve_expiry(uid.expiry, policy.now) { Some(at) => Expiry::At(at), None => Expiry::Never, @@ -141,11 +144,13 @@ fn stored_uid( return Ok(stored); }; // Unusable refresh state is dropped, never the UID: a refresh with no - // stated time, with empty or oversized state, or due already (which - // would make the source due on every navigation) is simply not stored. + // stated time or with empty or oversized state is simply not stored. + // State that is already due is kept, so a live UID imported after its + // refresh time, or held through a failed refresh, can still be + // refreshed; `rate_limit_per_hour` bounds the retries. let usable = !refresh.state.is_empty() && refresh.state.len() <= MAX_REFRESH_STATE_BYTES; match resolve_expiry(refresh.from, policy.now) { - Some(from) if usable && from > policy.now => Ok(stored.with_refresh(KvRefresh { + Some(from) if usable => Ok(stored.with_refresh(KvRefresh { state: refresh.state.into_owned(), from, })), @@ -164,8 +169,10 @@ fn stored_uid( fn resolve_expiry(expiry: ProviderExpiry, now: UnixSeconds) -> Option { match expiry { ProviderExpiry::Unspecified => None, - ProviderExpiry::At(at) => Some(at), - ProviderExpiry::After(after) => Some(now.saturating_add_secs(after.as_secs())), + ProviderExpiry::At(at) => Some(at.min(now.saturating_add_secs(MAX_EXPIRY_SECS))), + ProviderExpiry::After(after) => { + Some(now.saturating_add_secs(after.as_secs().min(MAX_EXPIRY_SECS))) + } } } @@ -262,7 +269,7 @@ mod tests { } #[test] - fn keeps_the_uid_when_its_refresh_state_is_unusable() { + fn keeps_due_refresh_state_and_drops_only_unusable_state() { let partners = registry(); let provider = ProviderId::new("example-provider"); let claimed = claimed(); @@ -279,17 +286,17 @@ mod tests { ( "state".to_owned(), ProviderExpiry::After(Duration::ZERO), - None, + Some(UnixSeconds::new(1000)), ), ( "state".to_owned(), ProviderExpiry::At(UnixSeconds::new(1000)), - None, + Some(UnixSeconds::new(1000)), ), ( "state".to_owned(), ProviderExpiry::At(UnixSeconds::new(999)), - None, + Some(UnixSeconds::new(999)), ), ]; @@ -311,11 +318,61 @@ mod tests { .refresh() .map(|refresh| refresh.from), expected, - "should keep only usable refresh state" + "should keep usable refresh state, due or not" ); } } + #[test] + fn clamps_relative_expiry_to_thirty_days() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + let mut far = uid("u-far"); + far.expiry = ProviderExpiry::After(Duration::from_secs(u64::MAX)); + + let (entries, dropped) = validate( + vec![eid("ids.example.com", vec![far])], + &policy(&provider, &claimed, &partners), + ); + + assert_eq!(dropped, DropCounts::default(), "should drop nothing"); + assert_eq!( + entries[0].1[0].uids()[0].expiry(), + Expiry::At(UnixSeconds::new(1000 + MAX_EXPIRY_SECS)), + "should clamp a far relative expiry to now plus 30 days" + ); + } + + #[test] + fn clamps_absolute_expiry_to_thirty_days() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + let mut far = uid("u-far"); + far.expiry = ProviderExpiry::At(UnixSeconds::new(u64::MAX)); + let mut near = uid("u-near"); + near.expiry = ProviderExpiry::At(UnixSeconds::new(5000)); + + let (entries, dropped) = validate( + vec![eid("ids.example.com", vec![far, near])], + &policy(&provider, &claimed, &partners), + ); + + assert_eq!(dropped, DropCounts::default(), "should drop nothing"); + let uids = entries[0].1[0].uids(); + assert_eq!( + uids[0].expiry(), + Expiry::At(UnixSeconds::new(1000 + MAX_EXPIRY_SECS)), + "should clamp a far expiry to now plus 30 days" + ); + assert_eq!( + uids[1].expiry(), + Expiry::At(UnixSeconds::new(5000)), + "should keep a near expiry" + ); + } + #[test] fn drops_invalid_values_by_reason() { let partners = registry(); @@ -326,7 +383,7 @@ mod tests { expired.expiry = ProviderExpiry::At(UnixSeconds::new(1000)); let array_ext = RawValue::from_string("[1]".to_owned()).expect("should build raw"); let mut bad_ext = uid("u-ext"); - bad_ext.ext = Some(&array_ext); + bad_ext.ext = Some(Cow::Borrowed(&array_ext)); let (entries, dropped) = validate( vec![ @@ -434,4 +491,25 @@ mod tests { "should keep valid mm" ); } + + #[test] + fn preserves_an_eid_ext_the_provider_built() { + let partners = registry(); + let provider = ProviderId::new("example-provider"); + let claimed = claimed(); + let mut built = eid("ids.example.com", vec![uid("a")]); + built.ext = Some(Cow::Owned( + RawValue::from_string(serde_json::json!({ "segments": ["s1"] }).to_string()) + .expect("should build raw"), + )); + + let (entries, dropped) = validate(vec![built], &policy(&provider, &claimed, &partners)); + + assert_eq!(dropped, DropCounts::default(), "should drop nothing"); + assert_eq!( + entries[0].1[0].fields().ext.as_ref().map(RawJson::to_value), + Some(serde_json::json!({ "segments": ["s1"] })), + "should keep the built ext" + ); + } } diff --git a/crates/trusted-server-core/src/ec/eids.rs b/crates/trusted-server-core/src/ec/eids.rs index 9e7a4bad6..f1073b244 100644 --- a/crates/trusted-server-core/src/ec/eids.rs +++ b/crates/trusted-server-core/src/ec/eids.rs @@ -105,7 +105,8 @@ impl KvEntry { /// source order, borrowing from the row. /// /// A source whose owning provider has a TCF vendor ID is skipped while - /// `consent` withholds that vendor's consent. + /// `consent` withholds that vendor's consent, and a source a provider + /// excludes where GDPR applies is skipped for a GDPR visitor. pub fn live_eids<'row>( &'row self, registry: &'row PartnerRegistry, @@ -114,7 +115,9 @@ impl KvEntry { ) -> impl Iterator> + 'row { self.ids.iter().flat_map(move |(source, record)| { let partner = registry.get(source).filter(|partner| { - partner.bidstream_enabled && !partner.owner_vendor_consent_denied(consent) + partner.bidstream_enabled + && !partner.owner_vendor_consent_denied(consent) + && !partner.excluded_for(consent) }); partner .into_iter() @@ -330,6 +333,38 @@ mod tests { ); } + #[test] + fn live_eids_withhold_a_gdpr_excluded_source_for_a_gdpr_visitor() { + // Arrange: a stored UID for a source a provider excludes where GDPR + // applies, written while the visitor was outside GDPR. + let mut registry = registry_with("ids.example.com", true); + assert!( + registry.exclude_where_gdpr_applies("ids.example.com"), + "should mark the source" + ); + let mut entry = KvEntry::minimal("ids.example.com", "placeholder", 1); + entry.ids.insert( + "ids.example.com".to_owned(), + provider_record(&[("p-1", Expiry::Never)]), + ); + let gdpr = consent_granting_vendors(vec![1]); + + // Act + let under_gdpr = entry + .live_eids(®istry, &gdpr, UnixSeconds::new(0)) + .count(); + let elsewhere = entry + .live_eids(®istry, &ConsentContext::default(), UnixSeconds::new(0)) + .count(); + + // Assert + assert_eq!( + under_gdpr, 0, + "should withhold the excluded source with TCF Purposes 1 and 4 granted" + ); + assert_eq!(elsewhere, 1, "should serve the source outside GDPR"); + } + #[test] fn live_eids_respect_bidstream_flag() { let registry = registry_with("ids.example.com", false); diff --git a/crates/trusted-server-core/src/ec/finalize.rs b/crates/trusted-server-core/src/ec/finalize.rs index b38541490..5c99b284d 100644 --- a/crates/trusted-server-core/src/ec/finalize.rs +++ b/crates/trusted-server-core/src/ec/finalize.rs @@ -566,7 +566,7 @@ mod tests { use crate::consent::types::{ConsentContext, ConsentSource}; use crate::ec::EidSyncSource; use crate::ec::coordinator::results::{ - DropCounts, LookupResults, LookupStatus, ProviderLookup, + DropCounts, LookupResults, LookupStatus, OptOutMemory, ProviderLookup, }; use crate::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, SourceDomain, UidValue}; use crate::ec::kv_types::{KvEid, KvUid}; @@ -2905,6 +2905,7 @@ mod tests { vec![KvEid::new(EidFields::default(), vec![uid])], )], cleared: Vec::new(), + opt_out: OptOutMemory::Unchanged, }], )); let mut response = empty_response(); diff --git a/crates/trusted-server-core/src/ec/kv_types.rs b/crates/trusted-server-core/src/ec/kv_types.rs index 3772f07f3..3a7409190 100644 --- a/crates/trusted-server-core/src/ec/kv_types.rs +++ b/crates/trusted-server-core/src/ec/kv_types.rs @@ -706,6 +706,11 @@ impl KvEntry { ); } + /// Forgets the opt-out remembered for `provider`, if any. + pub fn forget_opt_out(&mut self, provider: &ProviderId) { + self.opt_outs.remove(provider); + } + /// Forgets opt-outs whose time has passed. pub fn prune_opt_outs(&mut self, now: UnixSeconds) { self.opt_outs.retain(|_, until| *until > now.get()); diff --git a/crates/trusted-server-core/src/ec/provider.rs b/crates/trusted-server-core/src/ec/provider.rs index 9f479fa29..9c8c40b08 100644 --- a/crates/trusted-server-core/src/ec/provider.rs +++ b/crates/trusted-server-core/src/ec/provider.rs @@ -50,6 +50,10 @@ pub trait IdentityProvider: Send + Sync { /// Parses a provider response, borrowing from its body. /// + /// A `404` is reported as no match before this is called; every other + /// status, `204` included, is handed to the provider, which decides + /// whether it means no match, an opt-out or an error. + /// /// # Errors /// /// Returns [`ProviderError::InvalidResponse`] when the body cannot be @@ -70,6 +74,59 @@ pub trait IdentityProvider: Send + Sync { let _ = lookup; None } + + /// Reads IDs from the vendor's own first-party cookies on the request. + /// + /// Runs on every consented navigation, before the due check, so a + /// changed cookie or an opt-out is noticed even while the provider's + /// sources hold live UIDs. Returned entries go through the same + /// validation, consent gates and write plan as lookup results, and a + /// kept UID makes the provider skip its lookup for this navigation. + /// [`CookieRead::OptedOut`] skips the lookup and removes the provider's + /// stored UIDs. The default reads nothing. + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> CookieRead<'cookie> { + let _ = cookies; + CookieRead::Absent + } + + /// Whether `source`'s IDs must not be collected or kept where GDPR applies. + /// + /// Where the visitor's jurisdiction is GDPR, the coordinator neither looks + /// up nor stores an excluded source and removes the UIDs the row holds for + /// it. A provider whose every source is excluded is skipped entirely: its + /// cookies are not read and no lookup is made. The default excludes + /// nothing. + fn excluded_where_gdpr_applies(&self, source: &SourceDomain) -> bool { + let _ = source; + false + } + + /// Whether a stored UID that outlives the one a cookie carries is kept + /// over the cookie's. + /// + /// For a provider that refreshes its IDs server side, a refresh updates + /// the row but not the browser's cookie, so the next navigation's older + /// cookie would otherwise roll the refreshed UID back. When this returns + /// `true`, a cookie UID whose absolute expiry is earlier than a live + /// stored UID's is ignored; a cookie opt-out still wins. The default + /// always takes the cookie. + fn keeps_fresher_stored_uids(&self) -> bool { + false + } + + /// Whether a live UID the row already stores wins over a different one + /// in the provider's cookies. + /// + /// For a first-party ID that should stay stable for as long as the row + /// lives, such as Prebid's `SharedID`: when this returns `true`, a cookie + /// is taken only for a source with no live stored UID. A cookie opt-out + /// still wins. The default always takes the cookie. + fn keeps_stored_uids(&self) -> bool { + false + } } impl IdentityProvider for Box

{ @@ -94,6 +151,157 @@ impl IdentityProvider for Box

{ fn generate(&self, lookup: &LookupInput<'_>) -> Option>> { (**self).generate(lookup) } + + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> CookieRead<'cookie> { + (**self).read_request_cookies(cookies) + } + + fn excluded_where_gdpr_applies(&self, source: &SourceDomain) -> bool { + (**self).excluded_where_gdpr_applies(source) + } + + fn keeps_fresher_stored_uids(&self) -> bool { + (**self).keeps_fresher_stored_uids() + } + + fn keeps_stored_uids(&self) -> bool { + (**self).keeps_stored_uids() + } +} + +/// What a provider found in the vendor's own first-party cookies. +#[derive(Debug)] +pub enum CookieRead<'cookie> { + /// The cookies hold IDs. + Found(Vec>), + /// The cookies record that the user opted out of the vendor. + OptedOut, + /// The cookies hold no usable ID and no opt-out. + Absent, +} + +impl<'cookie> CookieRead<'cookie> { + /// The IDs found, or `None` for an opt-out or no ID. + #[must_use] + pub fn into_found(self) -> Option>> { + match self { + Self::Found(eids) => Some(eids), + Self::OptedOut | Self::Absent => None, + } + } +} + +impl<'cookie> From>>> for CookieRead<'cookie> { + fn from(eids: Option>>) -> Self { + eids.map_or(Self::Absent, Self::Found) + } +} + +/// The request's `Cookie` header, lent to providers that read their own +/// first-party cookies. +/// +/// It also carries the request time, so a provider judges a cookie's stated +/// expiry against the navigation's one clock reading rather than its own. +/// Its `Debug` output names the cookies but never prints their values. +#[derive(Clone, Copy)] +pub struct RequestCookies<'a> { + header: &'a str, + now: UnixSeconds, + gdpr_applies: bool, +} + +impl Default for RequestCookies<'_> { + fn default() -> Self { + Self::new("") + } +} + +impl core::fmt::Debug for RequestCookies<'_> { + fn fmt(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter + .debug_list() + .entries( + self.header + .split(';') + .filter_map(|pair| pair.split_once('=')) + .map(|(name, _)| name.trim()), + ) + .finish() + } +} + +impl<'a> RequestCookies<'a> { + /// Wraps a `Cookie` header value (pairs separated by `;`). + #[must_use] + pub fn new(header: &'a str) -> Self { + Self { + header, + now: UnixSeconds::new(0), + gdpr_applies: false, + } + } + + /// The same cookies, for a request where GDPR does or does not apply. + /// + /// The coordinator sets it from the request consent, for readers whose + /// cookie writers stay off where GDPR applies. + #[must_use] + pub fn where_gdpr_applies(self, gdpr_applies: bool) -> Self { + Self { + gdpr_applies, + ..self + } + } + + /// Whether GDPR applies to the request. + #[must_use] + pub fn gdpr_applies(&self) -> bool { + self.gdpr_applies + } + + /// The same cookies, read at request time `now`. + /// + /// The coordinator sets it before handing the cookies to a provider. + #[must_use] + pub fn at(self, now: UnixSeconds) -> Self { + Self { now, ..self } + } + + /// The request time the cookies are read at. + #[must_use] + pub fn now(&self) -> UnixSeconds { + self.now + } + + /// Returns the first non-empty value of cookie `name`, undecoded and + /// unquoted. + /// + /// Empty values are skipped, so an empty duplicate (another path or + /// domain) does not hide a later value; a name with only empty values + /// reads as absent. + #[must_use] + pub fn get(&self, name: &str) -> Option<&'a str> { + self.get_all(name).next() + } + + /// Every non-empty value of the cookie `name`, in header order. + /// + /// A request can carry several cookies with one name (set on different + /// paths or domains); opt-out checks must look at all of them. + pub fn get_all<'name>(&self, name: &'name str) -> impl Iterator + 'name + where + 'a: 'name, + { + self.header + .split(';') + .filter_map(|pair| pair.split_once('=')) + .filter(move |(key, _)| key.trim() == name) + .map(|(_, value)| value.trim().trim_matches('"')) + .filter(|value| !value.is_empty()) + } } /// What a provider can return, used by the coordinator to apply shared policy. @@ -158,6 +366,11 @@ pub struct LookupInput<'a> { pub due_sources: &'a [SourceDomain], /// This provider's currently stored entries, borrowed from the row. pub current: OwnedEntries<'a>, + /// The request's cookies, for a vendor first-party cookie a lookup + /// sends as input. + pub cookies: RequestCookies<'a>, + /// Request time; providers use it instead of reading the clock. + pub now: UnixSeconds, /// Always `None` in V1. pub hashed_email: Option<&'a HashedEmail>, } @@ -314,8 +527,8 @@ pub struct ProviderEid<'body> { pub matcher: Option>, /// `OpenRTB` `mm`. pub mm: Option, - /// EID-level `ext`. - pub ext: Option<&'body RawValue>, + /// EID-level `ext`, borrowed from the body or built by the provider. + pub ext: Option>, /// UIDs of the entry. pub uids: Vec>, } @@ -327,8 +540,8 @@ pub struct ProviderUid<'body> { pub id: Cow<'body, str>, /// Agent type; filled from partner config when missing. pub atype: Option, - /// UID-level `ext`. - pub ext: Option<&'body RawValue>, + /// UID-level `ext`, borrowed from the body or built by the provider. + pub ext: Option>, /// Expiry as the provider states it. pub expiry: ProviderExpiry, /// Private refresh state kept with the UID in the row and never served. @@ -404,8 +617,10 @@ pub trait ProviderKind { /// Shared configuration every provider receives at build time. #[derive(Debug, Clone, Copy)] pub struct ProviderCommon<'a> { - /// Configured lookup endpoint. - pub endpoint: &'a Url, + /// Configured provider ID, for logs that must tell instances apart. + pub id: &'a str, + /// Configured lookup endpoint, absent for cookie-only kinds. + pub endpoint: Option<&'a Url>, /// Configured sources. pub sources: &'a [SourceDomain], /// Resolved secrets. @@ -439,12 +654,11 @@ pub fn build_kind( let settings = K::Settings::deserialize(serde_json::Value::Object(config.settings.clone())) .change_context(ProviderConfigError::Invalid) .attach_with(|| format!("settings for provider `{}` are invalid", config.id))?; - let endpoint = Url::parse(&config.endpoint) - .change_context(ProviderConfigError::Invalid) - .attach_with(|| format!("endpoint for provider `{}` is not a URL", config.id))?; + let endpoint = parse_endpoint(config)?; let sources = parse_sources(config)?; let common = ProviderCommon { - endpoint: &endpoint, + id: &config.id, + endpoint: endpoint.as_ref(), sources: &sources, secrets, }; @@ -452,6 +666,21 @@ pub fn build_kind( Ok(Box::new(provider)) } +/// Parses a provider's configured endpoint, when one is set. +fn parse_endpoint( + config: &IdentityProviderConfig, +) -> Result, Report> { + config + .endpoint + .as_deref() + .map(|endpoint| { + Url::parse(endpoint) + .change_context(ProviderConfigError::Invalid) + .attach_with(|| format!("endpoint for provider `{}` is not a URL", config.id)) + }) + .transpose() +} + /// Parses and normalizes a provider's configured sources. pub(crate) fn parse_sources( config: &IdentityProviderConfig, @@ -478,7 +707,7 @@ pub(crate) fn destination_allowed>(url: &Url, allowed_domains: &[S } /// Checks the configuration rules core owns for every enabled provider -/// (spec §12.1): the endpoint is an `https` URL whose host is in +/// (spec §12.1): a configured endpoint is an `https` URL whose host is in /// `allowed_domains`, it claims at least one source, every source has an /// `[[ec.partners]]` entry, no source is claimed by two enabled providers, and /// every enabled provider has a non-empty id of its own. @@ -507,10 +736,9 @@ pub fn validate_enabled_configs<'a>( config.id ))); } - let endpoint = Url::parse(&config.endpoint) - .change_context(ProviderConfigError::Invalid) - .attach_with(|| format!("endpoint for provider `{}` is not a URL", config.id))?; - if !destination_allowed(&endpoint, &config.allowed_domains) { + let endpoint = parse_endpoint(config)?; + if endpoint.is_some_and(|endpoint| !destination_allowed(&endpoint, &config.allowed_domains)) + { return Err(Report::new(ProviderConfigError::Invalid).attach(format!( "endpoint for provider `{}` must be https with a host in allowed_domains", config.id @@ -701,6 +929,34 @@ mod tests { ); } + #[test] + fn cookies_get_all_returns_every_non_empty_duplicate() { + let cookies = RequestCookies::new("id=; other=x; id=first; id=second"); + + assert_eq!( + cookies.get_all("id").collect::>(), + vec!["first", "second"], + "should return every non-empty value in order" + ); + assert_eq!( + cookies.get("id"), + Some("first"), + "should get the first value" + ); + } + + #[test] + fn cookies_debug_prints_names_but_not_values() { + let cookies = RequestCookies::new("session=example-secret; vendor_id=v-1"); + + let printed = format!("{cookies:?}"); + + assert_eq!( + printed, r#"["session", "vendor_id"]"#, + "should print only cookie names" + ); + } + #[test] fn request_debug_hides_query_and_headers() { let mut headers = http::HeaderMap::new(); @@ -758,9 +1014,9 @@ mod tests { #[test] fn validate_enabled_configs_rejects_endpoint_outside_allowlist() { let mut outside = config(&json!({})); - outside.endpoint = "https://other.example.com/".to_owned(); + outside.endpoint = Some("https://other.example.com/".to_owned()); let mut plain_http = config(&json!({})); - plain_http.endpoint = "http://ids.example.com/".to_owned(); + plain_http.endpoint = Some("http://ids.example.com/".to_owned()); for rejected in [outside, plain_http] { let error = validate_enabled_configs([&rejected], &partners()) @@ -856,12 +1112,40 @@ mod tests { let first = config(&json!({})); let mut disabled = config(&json!({})); disabled.enabled = false; - disabled.endpoint = "https://other.example.com/".to_owned(); + disabled.endpoint = Some("https://other.example.com/".to_owned()); validate_enabled_configs([&first, &disabled], &partners()) .expect("should ignore disabled provider"); } + #[test] + fn validate_enabled_configs_accepts_cookie_only_provider_without_endpoint() { + let mut cookie_only = config(&json!({})); + cookie_only.endpoint = None; + + validate_enabled_configs([&cookie_only], &partners()) + .expect("should accept a provider without endpoint"); + } + + #[test] + fn request_cookies_finds_named_values() { + let cookies = RequestCookies::new("a=1; vendor_id=\"v-1\" ;empty=; vendor_id=v-2"); + + assert_eq!( + cookies.get("vendor_id"), + Some("v-1"), + "should find first value" + ); + assert_eq!(cookies.get("a"), Some("1"), "should find leading cookie"); + assert_eq!(cookies.get("empty"), None, "should ignore empty values"); + assert_eq!( + RequestCookies::new("vendor_id=; vendor_id=v-2").get("vendor_id"), + Some("v-2"), + "should skip an empty duplicate before a later value" + ); + assert_eq!(cookies.get("missing"), None, "should miss absent cookies"); + } + #[test] fn owned_entries_yields_only_configured_sources() { let entry: KvEntry = serde_json::from_value(json!({ diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index d5064a984..fc1f89a86 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -562,6 +562,7 @@ mod tests { ts_pull_token: Some(Redacted::new("token".to_owned())), provider_owned: false, owner_tcf_vendor_ids: Vec::new(), + excluded_where_gdpr_applies: false, } } diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index 4e840cd28..fe20f8312 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -9,6 +9,7 @@ use std::num::NonZeroU16; use error_stack::{Report, ResultExt as _}; +use crate::consent::jurisdiction::Jurisdiction; use crate::consent::{ConsentContext, tcf_vendor_consent_denied}; use crate::error::TrustedServerError; use crate::redacted::Redacted; @@ -56,6 +57,10 @@ pub struct PartnerConfig { /// claim this source. Its UIDs leave the server only while every one of /// these vendors has consent. pub owner_tcf_vendor_ids: Vec, + /// Whether a configured provider, enabled or disabled, excludes this + /// source where GDPR applies. Its UIDs then never leave the server for a + /// visitor whose jurisdiction is GDPR. + pub excluded_where_gdpr_applies: bool, } impl PartnerConfig { @@ -67,6 +72,13 @@ impl PartnerConfig { .iter() .any(|vendor| tcf_vendor_consent_denied(consent, *vendor)) } + + /// Whether this source's UIDs are withheld because a provider excludes it + /// and the visitor's jurisdiction is GDPR. + #[must_use] + pub fn excluded_for(&self, consent: &ConsentContext) -> bool { + self.excluded_where_gdpr_applies && matches!(consent.jurisdiction, Jurisdiction::Gdpr) + } } /// In-memory partner registry with O(1) lookups by source domain and API key hash. @@ -337,6 +349,23 @@ impl PartnerRegistry { } } + /// Marks `source` as excluded where GDPR applies, so its stored UIDs are + /// withheld from egress for a visitor whose jurisdiction is GDPR. + /// + /// Returns `false` when the source has no `[[ec.partners]]` entry. + pub fn exclude_where_gdpr_applies(&mut self, source: &str) -> bool { + let Ok(normalized) = normalize_partner_source_domain(source) else { + return false; + }; + match self.by_source_domain.get_mut(&normalized) { + Some(partner) => { + partner.excluded_where_gdpr_applies = true; + true + } + None => false, + } + } + /// Whether an enabled identity provider owns `source`. #[must_use] pub fn is_provider_owned(&self, source: &str) -> bool { @@ -421,6 +450,7 @@ fn build_partner_config( ts_pull_token: partner.ts_pull_token.clone(), provider_owned: false, owner_tcf_vendor_ids: Vec::new(), + excluded_where_gdpr_applies: false, } } diff --git a/crates/trusted-server-core/src/ec/write_plan.rs b/crates/trusted-server-core/src/ec/write_plan.rs index 23718cd32..c30e0e1cc 100644 --- a/crates/trusted-server-core/src/ec/write_plan.rs +++ b/crates/trusted-server-core/src/ec/write_plan.rs @@ -11,7 +11,7 @@ use std::collections::btree_map; use super::EcKvSnapshot; -use super::coordinator::results::{LookupResults, LookupStatus, SkipReason}; +use super::coordinator::results::{LookupResults, LookupStatus, OptOutMemory, SkipReason}; use super::eid_types::{SourceDomain, UnixSeconds}; use super::kv::PartnerIdUpdate; use super::kv_types::{KvEntry, KvPartnerId, MAX_ENTRY_BYTES}; @@ -130,9 +130,11 @@ fn apply_lookups( let mut written = Vec::new(); for lookup in lookups.iter() { // A newer request may have recorded this provider's opt-out after - // the lookup was dispatched; never restore its UIDs over it. + // the lookup was dispatched; never restore its UIDs over it, unless + // the vendor's cookies showed the visitor opted back in. if entry.opted_out(&lookup.provider, now) && lookup.status != LookupStatus::Skipped(SkipReason::OptedOut) + && lookup.opt_out != OptOutMemory::Forget { log::info!( "EC write planner: discarding results of opted-out provider '{}'", @@ -140,13 +142,16 @@ fn apply_lookups( ); continue; } + if lookup.opt_out == OptOutMemory::Forget { + entry.forget_opt_out(&lookup.provider); + } for (source, eids) in &lookup.entries { if let Some(record) = KvPartnerId::from_eids(eids.clone(), now) { entry.ids.insert(source.as_str().to_owned(), record); written.push(source.clone()); } } - if lookup.status == LookupStatus::Skipped(SkipReason::OptedOut) { + if lookup.opt_out == OptOutMemory::Remember { entry.remember_opt_out(&lookup.provider, now); } for source in &lookup.cleared { @@ -386,6 +391,7 @@ mod tests { vec![KvEid::new(EidFields::default(), uids)], )], cleared: Vec::new(), + opt_out: OptOutMemory::Unchanged, }], ) } @@ -609,6 +615,10 @@ mod tests { } fn failed_lookup(status: LookupStatus) -> LookupResults { + lookup_with_memory(status, OptOutMemory::Unchanged) + } + + fn lookup_with_memory(status: LookupStatus, opt_out: OptOutMemory) -> LookupResults { LookupResults::new( EC_ID, vec![ProviderLookup { @@ -616,10 +626,88 @@ mod tests { status, entries: Vec::new(), cleared: Vec::new(), + opt_out, }], ) } + #[test] + fn a_cookie_opt_out_is_not_remembered() { + // Arrange + let registry = registry(&["ids.example.com"]); + let ingest: [PartnerIdUpdate; 0] = []; + let mut lookups = failed_lookup(LookupStatus::Skipped(SkipReason::OptedOut)); + let cleared = lookups.iter().next().expect("should hold a lookup").clone(); + lookups = LookupResults::new( + EC_ID, + vec![ProviderLookup { + cleared: vec![SourceDomain::try_from("ids.example.com").expect("should parse")], + ..cleared + }], + ); + + // Act + let change = plan_write(&input( + &present(&provider_entry(&["p-1"])), + None, + &ingest, + &lookups, + ®istry, + )) + .expect("should plan update"); + + // Assert + let RowChange::Update { entry, .. } = change else { + panic!("should update existing row") + }; + assert!( + !entry.ids.contains_key("ids.example.com"), + "should clear the opted-out source" + ); + assert!( + entry.opt_outs.is_empty(), + "should not remember an opt-out read from cookies" + ); + } + + #[test] + fn an_opt_back_in_forgets_the_opt_out_and_stores_the_ids() { + // Arrange + let registry = registry(&["ids.example.com"]); + let ingest: [PartnerIdUpdate; 0] = []; + let provider = ProviderId::new("example-provider"); + let mut entry = live_entry(); + entry + .opt_outs + .insert(provider.clone(), 2000 + OPT_OUT_MEMORY_SECONDS); + let found = found("ids.example.com", &["c-1"]); + let lookup = found.iter().next().expect("should hold a lookup").clone(); + let lookups = LookupResults::new( + EC_ID, + vec![ProviderLookup { + opt_out: OptOutMemory::Forget, + ..lookup + }], + ); + + // Act + let change = plan_write(&input(&present(&entry), None, &ingest, &lookups, ®istry)) + .expect("should plan update"); + + // Assert + let RowChange::Update { entry, .. } = change else { + panic!("should update existing row") + }; + assert!( + !entry.opt_outs.contains_key(&provider), + "should forget the remembered opt-out" + ); + assert!( + entry.ids.contains_key("ids.example.com"), + "should store the cookie ID" + ); + } + #[test] fn opt_out_removes_the_provider_sources_and_blocks_browser_fallback() { // Arrange @@ -633,6 +721,7 @@ mod tests { status: LookupStatus::Skipped(SkipReason::OptedOut), entries: Vec::new(), cleared: vec![SourceDomain::try_from("ids.example.com").expect("should parse")], + opt_out: OptOutMemory::Unchanged, }], ); @@ -656,7 +745,10 @@ mod tests { let registry = registry(&["ids.example.com"]); let ingest: [PartnerIdUpdate; 0] = []; let provider = ProviderId::new("example-provider"); - let lookups = failed_lookup(LookupStatus::Skipped(SkipReason::OptedOut)); + let lookups = lookup_with_memory( + LookupStatus::Skipped(SkipReason::OptedOut), + OptOutMemory::Remember, + ); // Act: the first opt-out is remembered. let first = plan_write(&input( @@ -747,6 +839,7 @@ mod tests { status: LookupStatus::Skipped(SkipReason::NotDue), entries: Vec::new(), cleared: Vec::new(), + opt_out: OptOutMemory::Unchanged, }], ); diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 095b846ec..bf3418b84 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -581,8 +581,11 @@ impl EcPartner { } /// One `[[ec.identity_providers]]` entry. +/// +/// A `sharedid` provider with no `sources` claims `pubcid.org`, the source +/// Prebid sends `SharedID` under. #[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] +#[serde(remote = "Self", deny_unknown_fields)] pub struct IdentityProviderConfig { /// Unique provider ID; recorded as the owner of the UIDs it writes. pub id: String, @@ -595,7 +598,11 @@ pub struct IdentityProviderConfig { )] pub enabled: bool, /// Lookup endpoint; must be `https` with a host in `allowed_domains`. - pub endpoint: String, + /// + /// Optional for kinds that only read first-party cookies; kinds that + /// make lookups reject a missing endpoint when they are built. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub endpoint: Option, /// Exact hosts the coordinator may call for this provider. #[serde(default, deserialize_with = "vec_from_seq_or_map")] pub allowed_domains: Vec, @@ -626,6 +633,28 @@ pub struct IdentityProviderConfig { pub settings: serde_json::Map, } +impl<'de> Deserialize<'de> for IdentityProviderConfig { + fn deserialize>(deserializer: D) -> Result { + let mut config = Self::deserialize(deserializer)?; + if config.kind == SHAREDID_PROVIDER_KIND && config.sources.is_empty() { + config.sources = vec![SHAREDID_DEFAULT_SOURCE.to_owned()]; + } + Ok(config) + } +} + +impl Serialize for IdentityProviderConfig { + fn serialize(&self, serializer: S) -> Result { + Self::serialize(self, serializer) + } +} + +/// Provider kind of Prebid's `SharedID`. +const SHAREDID_PROVIDER_KIND: &str = "sharedid"; + +/// Source a `sharedid` provider claims when none is configured. +pub const SHAREDID_DEFAULT_SOURCE: &str = "pubcid.org"; + impl IdentityProviderConfig { const fn default_enabled() -> bool { true diff --git a/crates/trusted-server-identity-providers/Cargo.toml b/crates/trusted-server-identity-providers/Cargo.toml index 1ff50dedf..d267ebc66 100644 --- a/crates/trusted-server-identity-providers/Cargo.toml +++ b/crates/trusted-server-identity-providers/Cargo.toml @@ -14,8 +14,20 @@ doctest = false workspace = true [dependencies] +aes-gcm = { workspace = true } +base64 = { workspace = true } +chrono = { workspace = true } +bytes = { workspace = true } error-stack = { workspace = true } +http = { workspace = true } +httpdate = { workspace = true } +log = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true, features = ["raw_value"] } trusted-server-core = { workspace = true } +url = { workspace = true } +urlencoding = { workspace = true } +uuid = { workspace = true } [dev-dependencies] toml = { workspace = true } diff --git a/crates/trusted-server-identity-providers/src/common.rs b/crates/trusted-server-identity-providers/src/common.rs new file mode 100644 index 000000000..518878aa0 --- /dev/null +++ b/crates/trusted-server-identity-providers/src/common.rs @@ -0,0 +1,510 @@ +//! Helpers shared by the vendor provider modules. + +use std::borrow::Cow; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::num::NonZeroUsize; +use std::time::Duration; + +use error_stack::Report; +use serde::Deserialize; +use serde_json::value::RawValue; +use trusted_server_core::ec::eid_types::{SourceDomain, UnixSeconds}; +use trusted_server_core::ec::provider::{ + EidLevelFields, EidShape, ProviderCapabilities, ProviderCommon, ProviderConfigError, + ProviderEid, ProviderExpiry, ProviderKind, ProviderUid, RequestCookies, SourceCapability, + SourceInput, +}; + +/// Returns whether `ip` is a public address worth sending to a vendor. +/// +/// Drops every special-purpose range that identifies nothing outside the +/// operator's network or is never a real client: for IPv4 "this network" +/// `0.0.0.0/8`, private, carrier-grade NAT `100.64.0.0/10`, loopback, +/// link-local, IETF protocol assignments `192.0.0.0/24`, the TEST-NETs, +/// the 6to4 relay anycast `192.88.99.0/24`, benchmarking `198.18.0.0/15`, +/// multicast and the reserved `240.0.0.0/4` (including broadcast). For IPv6 +/// it drops unspecified, loopback, IPv4-compatible, discard-only +/// `100::/64`, IETF protocol assignments `2001::/23`, documentation +/// `2001:db8::/32`, unique-local `fc00::/7`, link-local `fe80::/10`, +/// site-local `fec0::/10` and multicast. IPv4-mapped and NAT64 +/// `64:ff9b::/96` addresses are judged by their embedded IPv4 address. +#[must_use] +pub(crate) fn is_routable_ip(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(v4) => is_routable_ipv4(v4), + IpAddr::V6(v6) => is_routable_ipv6(v6), + } +} + +fn is_routable_ipv4(v4: Ipv4Addr) -> bool { + let [first, second, third, _] = v4.octets(); + let this_network = first == 0; + let carrier_grade_nat = first == 100 && (second & 0xc0) == 64; + let protocol_assignments = first == 192 && second == 0 && third == 0; + let test_net = matches!( + (first, second, third), + (192, 0, 2) | (198, 51, 100) | (203, 0, 113) + ); + let relay_anycast = first == 192 && second == 88 && third == 99; + let benchmarking = first == 198 && (second & 0xfe) == 18; + let multicast = (first & 0xf0) == 224; + let reserved = (first & 0xf0) == 240; + !(this_network + || v4.is_loopback() + || v4.is_private() + || v4.is_link_local() + || carrier_grade_nat + || protocol_assignments + || test_net + || relay_anycast + || benchmarking + || multicast + || reserved) +} + +fn is_routable_ipv6(v6: Ipv6Addr) -> bool { + if let Some(v4) = v6.to_ipv4_mapped() { + return is_routable_ipv4(v4); + } + let segments = v6.segments(); + if segments[..6] == [0x64, 0xff9b, 0, 0, 0, 0] { + let [.., a, b, c, d] = v6.octets(); + return is_routable_ipv4(Ipv4Addr::new(a, b, c, d)); + } + let first = segments[0]; + let ipv4_compatible = segments[..6] == [0; 6]; + let discard_only = segments[..4] == [0x100, 0, 0, 0]; + let protocol_assignments = first == 0x2001 && segments[1] < 0x200; + let documentation = first == 0x2001 && segments[1] == 0x0db8; + let unique_local = (first & 0xfe00) == 0xfc00; + let link_local = (first & 0xffc0) == 0xfe80; + let site_local = (first & 0xffc0) == 0xfec0; + let multicast = (first & 0xff00) == 0xff00; + !(ipv4_compatible + || discard_only + || protocol_assignments + || documentation + || unique_local + || link_local + || site_local + || multicast) +} + +/// TTL given to a vendor UID whose response states no expiry. +pub(crate) const DEFAULT_TTL: Duration = Duration::from_secs(24 * 60 * 60); + +/// Shortest TTL a vendor UID is kept for. +pub(crate) const MIN_TTL: Duration = Duration::from_secs(60 * 60); + +/// Longest TTL a vendor UID is kept for. +pub(crate) const MAX_TTL: Duration = Duration::from_secs(30 * 24 * 60 * 60); + +/// Turns a vendor-stated TTL into a bounded relative expiry. +/// +/// A missing TTL becomes [`DEFAULT_TTL`] so a UID never lives forever. A +/// stated TTL is clamped to [`MIN_TTL`]..=[`MAX_TTL`]: a zero, past or very +/// short TTL keeps the UID for [`MIN_TTL`] instead of discarding a valid +/// ID, and the minimum still forces a refresh soon after. +#[must_use] +pub(crate) fn bounded_expiry(ttl: Option) -> ProviderExpiry { + ProviderExpiry::After(ttl.map_or(DEFAULT_TTL, |ttl| ttl.clamp(MIN_TTL, MAX_TTL))) +} + +/// Joins GPP section IDs with commas, as vendors expect them. +#[must_use] +pub(crate) fn join_section_ids(ids: &[u16]) -> String { + ids.iter().map(u16::to_string).collect::>().join(",") +} + +/// Agent type of the person-based IDs read from vendor cookies. +pub(crate) const COOKIE_ATYPE: i32 = 3; + +/// Builds the capabilities of a cookie-only provider, which claims exactly +/// one source and stores one UID for it. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError::Invalid`] unless exactly one source is +/// configured. +pub(crate) fn cookie_capabilities( + common: &ProviderCommon<'_>, + kind: &str, +) -> Result<(ProviderCapabilities, SourceDomain), Report> { + let [source] = common.sources else { + return Err(Report::new(ProviderConfigError::Invalid) + .attach(format!("a {kind} provider needs exactly one source"))); + }; + let one = NonZeroUsize::MIN; + let capabilities = ProviderCapabilities { + sources: Box::new([SourceCapability { + source: source.clone(), + input: SourceInput::Anonymous, + }]), + shape: EidShape { + max_eids: one, + max_uids_per_eid: one, + eid_level_fields: EidLevelFields::Ignored, + }, + }; + Ok((capabilities, source.clone())) +} + +/// Whether a cookie-carried ID is worth storing: not blank and not a +/// placeholder such as `0` or an all-zero UUID. +/// +/// A placeholder reads as "no ID", not as an opt-out: no vendor read with +/// this check documents it as its opt-out signal. Providers whose vendor +/// does (ID5's `"0"`) check for it separately and report an opt-out. +#[must_use] +pub(crate) fn is_usable_id(id: &str) -> bool { + let id = id.trim(); + !id.is_empty() && !id.chars().all(|c| c == '0' || c == '-') +} + +/// One EID for `source` holding the single UID `id` with agent type 3. +#[must_use] +pub(crate) fn cookie_eid<'cookie>( + source: &SourceDomain, + id: Cow<'cookie, str>, + expiry: ProviderExpiry, +) -> ProviderEid<'cookie> { + ProviderEid { + source: Cow::Owned(source.as_str().to_owned()), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids: vec![ProviderUid { + id, + atype: Some(COOKIE_ATYPE), + ext: None, + expiry, + refresh: None, + }], + } +} + +/// Agent type of the device-based IDs read from browser-ID cookies. +pub(crate) const DEVICE_ATYPE: i32 = 1; + +/// One EID for `source` holding the single device-based UID `id` with +/// agent type 1. +#[must_use] +pub(crate) fn device_eid<'cookie>( + source: &SourceDomain, + id: Cow<'cookie, str>, + expiry: ProviderExpiry, +) -> ProviderEid<'cookie> { + let mut eid = cookie_eid(source, id, expiry); + for uid in &mut eid.uids { + uid.atype = Some(DEVICE_ATYPE); + } + eid +} + +/// Percent-decodes a cookie value, keeping it unchanged when it is not +/// valid percent-encoded UTF-8. +#[must_use] +pub(crate) fn percent_decoded(value: &str) -> Cow<'_, str> { + urlencoding::decode(value).unwrap_or(Cow::Borrowed(value)) +} + +/// Settings for the Lockr capability, under a provider's +/// `[ec.identity_providers.settings.lockr]`. +/// +/// Lockr AIM writes other vendors' IDs into first-party cookies on the +/// publisher domain, under key names configured per publisher in Lockr. +/// Only kinds implementing [`ReadsLockr`] accept this table; the settings +/// of every other kind reject it as an unknown field. +#[derive(Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LockrSettings { + /// Whether the Lockr cookies are read. Off by default. + #[serde(default)] + pub enabled: bool, + /// Lockr cookies read in order, after the vendor's own cookies. + /// Defaults to the vendor's known Lockr key. + #[serde(default)] + pub cookie_names: Option>, +} + +/// The Lockr capability: declared by provider kinds whose IDs Lockr AIM +/// delivers. +pub(crate) trait ReadsLockr: ProviderKind { + /// The key names Lockr uses for this vendor by default. + const LOCKR_COOKIE_NAMES: &'static [&'static str]; +} + +/// Reads a provider's cookie-carried ID: the vendor's own cookies in order, +/// then, when the Lockr capability is enabled, the Lockr cookies in order. +/// +/// Each value goes to the provider's own parser, so Lockr reuses it. +#[derive(Debug)] +pub(crate) struct CookieReader { + provider_id: String, + own: Box<[String]>, + lockr: Box<[String]>, +} + +impl CookieReader { + /// A reader for a kind without the Lockr capability. + pub(crate) fn own(common: &ProviderCommon<'_>, own: Vec) -> Self { + Self { + provider_id: common.id.to_owned(), + own: own.into_boxed_slice(), + lockr: Box::default(), + } + } + + /// A reader for a kind with the Lockr capability. Lockr names already + /// in `own` are skipped, since they were read first. + /// + /// # Errors + /// + /// Returns [`ProviderConfigError::Invalid`] when Lockr is enabled with + /// an empty `cookie_names` list. + pub(crate) fn with_lockr( + common: &ProviderCommon<'_>, + own: Vec, + lockr: LockrSettings, + ) -> Result> { + let mut reader = Self::own(common, own); + if !lockr.enabled { + return Ok(reader); + } + let names = lockr.cookie_names.unwrap_or_else(|| { + K::LOCKR_COOKIE_NAMES + .iter() + .map(|name| (*name).to_owned()) + .collect() + }); + if names.is_empty() { + return Err(Report::new(ProviderConfigError::Invalid).attach(format!( + "lockr for a {} provider needs at least one cookie name", + K::KIND + ))); + } + reader.lockr = names + .into_iter() + .filter(|name| !reader.own.contains(name)) + .collect(); + Ok(reader) + } + + /// The Lockr cookie names to read for this request: none where GDPR + /// applies, because Lockr AIM does not run there, so a Lockr cookie on + /// such a request is a leftover rather than a current delivery. + fn lockr_names<'reader>( + &'reader self, + cookies: RequestCookies<'_>, + ) -> impl Iterator + 'reader { + let names: &[String] = if cookies.gdpr_applies() { + &[] + } else { + &self.lockr + }; + names.iter() + } + + /// Whether any cookie name is configured. + pub(crate) fn is_empty(&self) -> bool { + self.own.is_empty() && self.lockr.is_empty() + } + + /// Whether any configured cookie, own or Lockr, has a value matching + /// `matches`. + pub(crate) fn any(&self, cookies: RequestCookies<'_>, matches: impl Fn(&str) -> bool) -> bool { + self.own + .iter() + .chain(self.lockr_names(cookies)) + .flat_map(|name| cookies.get_all(name)) + .any(matches) + } + + /// Returns the first value `parse` accepts, trying every value of each + /// own cookie before the Lockr cookies, so an unusable or expired value + /// never hides a later usable one. + pub(crate) fn read<'cookie, T>( + &self, + cookies: RequestCookies<'cookie>, + mut parse: impl FnMut(&'cookie str) -> Option, + ) -> Option { + if let Some(found) = self + .own + .iter() + .flat_map(|name| cookies.get_all(name)) + .find_map(&mut parse) + { + return Some(found); + } + let (name, found) = self.lockr_names(cookies).find_map(|name| { + let found = cookies.get_all(name).find_map(&mut parse)?; + Some((name, found)) + })?; + log::debug!( + "Identity provider '{}' reads its ID from cookie '{name}' (origin lockr)", + self.provider_id + ); + Some(found) + } +} + +/// Whether a cookie's stated expiry has passed at request time `now`. +/// +/// Cookie readers skip such a value so it never hides a later live one. +pub(crate) fn is_expired(expiry: &ProviderExpiry, now: UnixSeconds) -> bool { + matches!(expiry, ProviderExpiry::At(at) if *at <= now) +} + +/// A complete `OpenRTB` EID as a vendor returns it, borrowed from the body. +/// +/// Unknown fields are ignored so one new vendor field does not fail the +/// whole response. +#[derive(Debug, Deserialize)] +pub(crate) struct VendorEid<'body> { + #[serde(borrow)] + pub source: Cow<'body, str>, + #[serde(default, borrow)] + pub uids: Vec>, + #[serde(default, borrow)] + pub inserter: Option>, + #[serde(default, borrow)] + pub matcher: Option>, + #[serde(default)] + pub mm: Option, + #[serde(default, borrow)] + pub ext: Option<&'body RawValue>, +} + +/// One UID of a [`VendorEid`]. +#[derive(Debug, Deserialize)] +pub(crate) struct VendorUid<'body> { + #[serde(borrow)] + pub id: Cow<'body, str>, + #[serde(default)] + pub atype: Option, + #[serde(default, borrow)] + pub ext: Option<&'body RawValue>, +} + +impl<'body> VendorEid<'body> { + /// Converts to a [`ProviderEid`], giving every UID `expiry`. + pub(crate) fn into_provider_eid(self, expiry: ProviderExpiry) -> ProviderEid<'body> { + ProviderEid { + source: self.source, + inserter: self.inserter, + matcher: self.matcher, + mm: self.mm, + ext: self.ext.map(Cow::Borrowed), + uids: self + .uids + .into_iter() + .map(|uid| ProviderUid { + id: uid.id, + atype: uid.atype, + ext: uid.ext.map(Cow::Borrowed), + expiry, + refresh: None, + }) + .collect(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn keeps_public_addresses() { + for ip in [ + "11.22.33.44", + "203.0.112.9", + "2a00:1450::1", + "::ffff:11.22.33.44", + ] { + assert!( + is_routable_ip(ip.parse().expect("should parse")), + "should keep {ip}" + ); + } + } + + #[test] + fn drops_special_purpose_addresses() { + for ip in [ + "0.1.2.3", + "127.0.0.1", + "10.1.2.3", + "172.16.0.1", + "192.168.1.1", + "169.254.1.1", + "100.64.0.1", + "100.127.255.254", + "192.0.0.8", + "192.0.2.10", + "198.51.100.7", + "203.0.113.5", + "192.88.99.1", + "198.18.0.1", + "198.19.255.1", + "224.0.0.1", + "239.1.1.1", + "240.0.0.1", + "255.255.255.255", + "::", + "::1", + "::10.1.2.3", + "100::1", + "2001::1", + "2001:db8::1", + "fc00::1", + "fd00::1", + "fe80::1", + "febf::1", + "fec0::1", + "ff02::1", + "::ffff:127.0.0.1", + "::ffff:100.64.0.1", + "64:ff9b::10.1.2.3", + ] { + assert!( + !is_routable_ip(ip.parse().expect("should parse")), + "should drop {ip}" + ); + } + } + + #[test] + fn filters_blank_and_opt_out_ids() { + for id in ["", " ", "0", "00000000-0000-0000-0000-000000000000"] { + assert!(!is_usable_id(id), "should drop {id:?}"); + } + assert!(is_usable_id("id-example-1"), "should keep a real ID"); + } + + #[test] + fn percent_decodes_leniently() { + assert_eq!(percent_decoded("a%20b"), "a b", "should decode"); + assert_eq!( + percent_decoded("a+b%zz"), + "a+b%zz", + "should keep invalid input" + ); + } + + #[test] + fn bounds_vendor_expiry() { + for (ttl, expected) in [ + (None, DEFAULT_TTL), + (Some(Duration::ZERO), MIN_TTL), + (Some(Duration::from_secs(7200)), Duration::from_secs(7200)), + (Some(Duration::from_secs(u64::MAX)), MAX_TTL), + ] { + assert!( + matches!(bounded_expiry(ttl), ProviderExpiry::After(after) if after == expected), + "should bound {ttl:?} to {expected:?}" + ); + } + } +} diff --git a/crates/trusted-server-identity-providers/src/connectid.rs b/crates/trusted-server-identity-providers/src/connectid.rs new file mode 100644 index 000000000..e33fe75cd --- /dev/null +++ b/crates/trusted-server-identity-providers/src/connectid.rs @@ -0,0 +1,308 @@ +//! Yahoo `ConnectID` identity provider: reads `ConnectID` from Yahoo's own +//! first-party cookie. +//! +//! Yahoo's `ConnectID` module stores a JSON object in the `connectId` cookie +//! (optionally percent-encoded) with `connectId`, `lastSynced` and `ttl`, +//! both in milliseconds. The ID is stored with agent type 3 until +//! `lastSynced + ttl`, with the TTL capped at [`MAX_TTL`]; the coordinator +//! drops it once that time has passed. As in Prebid's `connectIdSystem`, a +//! missing or zero `ttl` means [`MAX_TTL`], a missing `lastSynced` means +//! the ID is fresh (stored for [`MAX_TTL`]), and the key may also be spelled +//! `connectid`. No server call is made. +//! +//! Yahoo's opt-out (`connectIdOptOut` = `1`) is kept in local storage, which +//! a server never sees, so it is not honoured here: a `ConnectID` cookie left +//! behind after an opt-out is still read until it expires. +//! +//! This kind has no Lockr capability: Lockr AIM is not documented to +//! deliver `ConnectID`, so a `lockr` settings table is rejected. +//! +//! The cookie's `he` and `puid` fields are never read, so they are never +//! stored. Parsing is lenient: an undecodable cookie, a missing +//! `connectId`, or a blank or placeholder (`0`) ID reads as no ID. + +use std::borrow::Cow; +use std::time::Duration; + +use error_stack::Report; +use serde::Deserialize; +use trusted_server_core::ec::eid_types::{SourceDomain, UnixSeconds}; +use trusted_server_core::ec::provider::{ + CookieRead, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, + ProviderConfigError, ProviderError, ProviderExpiry, ProviderKind, ProviderRequest, + ProviderResponse, RequestCookies, +}; + +use crate::common::{ + CookieReader, cookie_capabilities, cookie_eid, is_expired, is_usable_id, percent_decoded, +}; + +/// Longest time a `ConnectID` is kept after its last sync. +pub const MAX_TTL: Duration = Duration::from_secs(14 * 24 * 60 * 60); + +/// The Yahoo `ConnectID` provider kind. +pub struct ConnectId; + +/// Settings for a `connectid` provider. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ConnectIdSettings { + /// Cookies holding the `ConnectID` object, read in order; the first + /// holding a usable ID wins. + #[serde(default = "ConnectIdSettings::default_cookie_names")] + pub cookie_names: Vec, +} + +impl ConnectIdSettings { + fn default_cookie_names() -> Vec { + vec!["connectId".to_owned()] + } +} + +/// A built Yahoo `ConnectID` provider. +#[derive(Debug)] +pub struct ConnectIdProvider { + capabilities: ProviderCapabilities, + source: SourceDomain, + cookies: CookieReader, +} + +impl ProviderKind for ConnectId { + const KIND: &'static str = "connectid"; + type Settings = ConnectIdSettings; + type Provider = ConnectIdProvider; + + fn build( + common: &ProviderCommon<'_>, + settings: ConnectIdSettings, + ) -> Result> { + if settings.cookie_names.is_empty() { + return Err(Report::new(ProviderConfigError::Invalid) + .attach("a connectid provider needs at least one cookie name")); + } + let (capabilities, source) = cookie_capabilities(common, Self::KIND)?; + Ok(ConnectIdProvider { + capabilities, + source, + cookies: CookieReader::own(common, settings.cookie_names), + }) + } +} + +/// The fields of the `connectId` cookie this provider reads. +#[derive(Deserialize)] +struct ConnectIdCookie { + #[serde(rename = "connectId", alias = "connectid")] + connect_id: String, + #[serde(rename = "lastSynced", default)] + last_synced_ms: Option, + #[serde(default)] + ttl: Option, +} + +/// Milliseconds as a duration; negative or non-finite values are zero. +fn from_millis(ms: f64) -> Duration { + Duration::try_from_secs_f64(ms / 1000.0).unwrap_or(Duration::ZERO) +} + +/// Decodes one cookie value into the ID and its expiry. +fn connect_id(value: &str, now: UnixSeconds) -> Option<(String, ProviderExpiry)> { + let decoded = percent_decoded(value); + let cookie: ConnectIdCookie = serde_json::from_str(&decoded).ok()?; + if !is_usable_id(&cookie.connect_id) { + return None; + } + let Some(last_synced_ms) = cookie.last_synced_ms else { + return Some((cookie.connect_id, ProviderExpiry::After(MAX_TTL))); + }; + let ttl = cookie + .ttl + .filter(|ttl| *ttl > 0.0) + .map_or(MAX_TTL, from_millis) + .min(MAX_TTL); + let expires = from_millis(last_synced_ms).saturating_add(ttl); + let expiry = ProviderExpiry::At(UnixSeconds::new(expires.as_secs())); + // An expired cookie never hides a later live one. + (!is_expired(&expiry, now)).then_some((cookie.connect_id, expiry)) +} + +impl IdentityProvider for ConnectIdProvider { + fn capabilities(&self) -> &ProviderCapabilities { + &self.capabilities + } + + fn build_request( + &self, + _lookup: &LookupInput<'_>, + ) -> Result> { + Err(Report::new(ProviderError::Request).attach("connectid makes no lookup")) + } + + fn parse_response<'body>( + &self, + _response: ProviderResponse<'body>, + ) -> Result, Report> { + Err(Report::new(ProviderError::InvalidResponse).attach("connectid makes no lookup")) + } + + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> CookieRead<'cookie> { + let Some((id, expiry)) = self + .cookies + .read(cookies, |value| connect_id(value, cookies.now())) + else { + return CookieRead::Absent; + }; + CookieRead::Found(vec![cookie_eid(&self.source, Cow::Owned(id), expiry)]) + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + use trusted_server_core::ec::provider::build_kind; + use trusted_server_core::settings::{IdentityProviderConfig, ProviderSecrets}; + + use super::*; + + /// Request time the tests read cookies at. + const NOW: UnixSeconds = UnixSeconds::new(1_700_000_000); + + fn provider() -> Box { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "connectid", + "kind": "connectid", + "sources": ["connectid.example.com"], + })) + .expect("should parse provider config"); + build_kind::(&config, &ProviderSecrets::default()) + .expect("should build provider") + } + + fn read(cookie: &serde_json::Value) -> Option<(String, ProviderExpiry)> { + let header = format!("connectId={}", urlencoding::encode(&cookie.to_string())); + let eids = provider() + .read_request_cookies(RequestCookies::new(&header).at(NOW)) + .into_found()?; + assert_eq!( + eids[0].source, "connectid.example.com", + "should use the source" + ); + let uid = &eids[0].uids[0]; + assert_eq!(uid.atype, Some(3), "should be person-based"); + Some((uid.id.to_string(), uid.expiry)) + } + + fn expires_at(read: Option<(String, ProviderExpiry)>) -> u64 { + match read.expect("should read the ID").1 { + ProviderExpiry::At(at) => at.get(), + other => panic!("should expire at a time, got {other:?}"), + } + } + + #[test] + fn reads_the_id_until_last_sync_plus_ttl() { + let cookie = json!({ + "connectId": "cid-example-1", + "he": "example-hashed-email", + "puid": "example-puid", + "lastSynced": 1_800_000_000_000_u64, + "lastUsed": 4_700_000_100_000_u64, + "ttl": 86_400_000, + }); + + let read = read(&cookie); + + assert_eq!( + read.as_ref().map(|(id, _)| id.as_str()), + Some("cid-example-1"), + "should store only connectId" + ); + assert_eq!( + expires_at(read), + 1_800_086_400, + "should expire at lastSynced + ttl" + ); + } + + #[test] + fn caps_the_ttl_and_defaults_a_missing_or_zero_one() { + // A far-future sync keeps the cookie live whatever today's date is. + let synced_secs = 1_800_000_000_u64; + let long = json!({ "connectId": "cid", "lastSynced": synced_secs * 1000, "ttl": 1e15 }); + let missing = json!({ "connectId": "cid", "lastSynced": synced_secs * 1000 }); + let zero = json!({ "connectId": "cid", "lastSynced": synced_secs * 1000, "ttl": 0 }); + + assert_eq!( + expires_at(read(&long)), + synced_secs + MAX_TTL.as_secs(), + "should cap at 14 days" + ); + for cookie in [missing, zero] { + assert_eq!( + expires_at(read(&cookie)), + synced_secs + MAX_TTL.as_secs(), + "should use Prebid's 14 days for {cookie}" + ); + } + } + + #[test] + fn reads_a_lowercase_key_and_a_missing_last_sync_as_fresh() { + let lowercase = json!({ "connectid": "cid", "lastSynced": 1_800_000_000_000_u64 }); + let unsynced = json!({ "connectId": "cid" }); + + assert_eq!( + read(&lowercase).map(|(id, _)| id).as_deref(), + Some("cid"), + "should accept the lowercase key" + ); + assert!( + matches!(read(&unsynced), Some((_, ProviderExpiry::After(ttl))) if ttl == MAX_TTL), + "should store an unsynced ID for 14 days" + ); + } + + #[test] + fn an_expired_cookie_does_not_hide_a_later_live_one() { + let expired = json!({ "connectId": "cid-old", "lastSynced": 1000, "ttl": 1000 }); + let live = json!({ "connectId": "cid-live", "lastSynced": 1_800_000_000_000_u64 }); + let header = format!( + "connectId={}; connectId={}", + urlencoding::encode(&expired.to_string()), + urlencoding::encode(&live.to_string()) + ); + + let eids = provider() + .read_request_cookies(RequestCookies::new(&header).at(NOW)) + .into_found() + .expect("should read the live cookie"); + + assert_eq!( + eids[0].uids[0].id, "cid-live", + "should skip the expired cookie" + ); + } + + #[test] + fn ignores_missing_malformed_and_opt_out_cookies() { + for cookie in [ + json!({}), + json!({ "connectId": "", "lastSynced": 1 }), + json!({ "connectId": "0", "lastSynced": 1 }), + json!({ "connectId": 5, "lastSynced": 1 }), + json!("not an object"), + ] { + assert!(read(&cookie).is_none(), "should ignore {cookie}"); + } + assert!( + provider() + .read_request_cookies(RequestCookies::new("connectId={broken").at(NOW)) + .into_found() + .is_none(), + "should ignore malformed JSON" + ); + } +} diff --git a/crates/trusted-server-identity-providers/src/criteo.rs b/crates/trusted-server-identity-providers/src/criteo.rs new file mode 100644 index 000000000..0d77c0023 --- /dev/null +++ b/crates/trusted-server-identity-providers/src/criteo.rs @@ -0,0 +1,184 @@ +//! Criteo identity provider: reads the Criteo ID from the first-party +//! cookie Prebid's `criteoIdSystem` writes. +//! +//! Prebid stores Criteo's bid ID in `cto_bidid` with `encodeURIComponent` +//! and sends the decoded value as the `criteo.com` EID with agent type 1, +//! so the cookie is percent-decoded before it is stored. Like Prebid's +//! `normalizeBidId`, a `{"criteoId":…}` value is unwrapped. No server call +//! is made. + +use std::borrow::Cow; + +use error_stack::Report; +use serde::Deserialize; +use trusted_server_core::ec::eid_types::SourceDomain; +use trusted_server_core::ec::provider::{ + CookieRead, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, + ProviderConfigError, ProviderError, ProviderKind, ProviderRequest, ProviderResponse, + RequestCookies, +}; + +use crate::common::{ + CookieReader, bounded_expiry, cookie_capabilities, device_eid, is_usable_id, percent_decoded, +}; + +/// The Criteo provider kind. +pub struct Criteo; + +/// Settings for a `criteo` provider. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CriteoSettings { + /// Cookies holding the Criteo bid ID, read in order. + #[serde(default = "CriteoSettings::default_cookie_names")] + pub cookie_names: Vec, +} + +impl CriteoSettings { + fn default_cookie_names() -> Vec { + vec!["cto_bidid".to_owned()] + } +} + +/// A built Criteo provider. +#[derive(Debug)] +pub struct CriteoProvider { + capabilities: ProviderCapabilities, + source: SourceDomain, + cookies: CookieReader, +} + +impl ProviderKind for Criteo { + const KIND: &'static str = "criteo"; + type Settings = CriteoSettings; + type Provider = CriteoProvider; + + fn build( + common: &ProviderCommon<'_>, + settings: CriteoSettings, + ) -> Result> { + if settings.cookie_names.is_empty() { + return Err(Report::new(ProviderConfigError::Invalid) + .attach("a criteo provider needs at least one cookie name")); + } + let (capabilities, source) = cookie_capabilities(common, Self::KIND)?; + Ok(CriteoProvider { + capabilities, + source, + cookies: CookieReader::own(common, settings.cookie_names), + }) + } +} + +impl IdentityProvider for CriteoProvider { + fn capabilities(&self) -> &ProviderCapabilities { + &self.capabilities + } + + fn build_request( + &self, + _lookup: &LookupInput<'_>, + ) -> Result> { + Err(Report::new(ProviderError::Request).attach("criteo makes no lookup")) + } + + fn parse_response<'body>( + &self, + _response: ProviderResponse<'body>, + ) -> Result, Report> { + Err(Report::new(ProviderError::InvalidResponse).attach("criteo makes no lookup")) + } + + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> CookieRead<'cookie> { + let Some(id) = self.cookies.read(cookies, |value| { + let id = bid_id(&percent_decoded(value))?; + is_usable_id(&id).then(|| id.trim().to_owned()) + }) else { + return CookieRead::Absent; + }; + CookieRead::Found(vec![device_eid( + &self.source, + Cow::Owned(id), + bounded_expiry(None), + )]) + } +} + +/// The bid ID in a decoded `cto_bidid` value, unwrapping the +/// `{"criteoId":…}` form Prebid's `normalizeBidId` accepts. +fn bid_id(decoded: &str) -> Option { + if decoded.trim_start().starts_with('{') { + let value: serde_json::Value = serde_json::from_str(decoded).ok()?; + return value["criteoId"].as_str().map(str::to_owned); + } + Some(decoded.to_owned()) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + use trusted_server_core::ec::provider::build_kind; + use trusted_server_core::settings::{IdentityProviderConfig, ProviderSecrets}; + + use super::*; + + fn provider() -> Box { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "criteo", + "kind": "criteo", + "sources": ["criteo.example.com"], + })) + .expect("should parse provider config"); + build_kind::(&config, &ProviderSecrets::default()).expect("should build provider") + } + + #[test] + fn stores_the_decoded_bid_id() { + let eids = provider() + .read_request_cookies(RequestCookies::new("cto_bidid=bidid-example%3D%3D")) + .into_found() + .expect("should read the bid ID"); + + assert_eq!( + eids[0].source, "criteo.example.com", + "should use the source" + ); + let uid = &eids[0].uids[0]; + assert_eq!( + uid.id, "bidid-example==", + "should store the decoded value Prebid sends" + ); + assert_eq!(uid.atype, Some(1), "should be device-based"); + } + + #[test] + fn unwraps_a_criteo_id_object() { + let eids = provider() + .read_request_cookies(RequestCookies::new( + "cto_bidid=%7B%22criteoId%22%3A%22bidid-example%22%7D", + )) + .into_found() + .expect("should read the bid ID"); + + assert_eq!( + eids[0].uids[0].id, "bidid-example", + "should store the wrapped bid ID" + ); + } + + #[test] + fn ignores_missing_and_blank_cookies() { + for header in ["cto_bidid=", "cto_bidid=0", "cto_bundle=bundle-example"] { + assert!( + provider() + .read_request_cookies(RequestCookies::new(header)) + .into_found() + .is_none(), + "should ignore {header}" + ); + } + } +} diff --git a/crates/trusted-server-identity-providers/src/id5.rs b/crates/trusted-server-identity-providers/src/id5.rs new file mode 100644 index 000000000..538190981 --- /dev/null +++ b/crates/trusted-server-identity-providers/src/id5.rs @@ -0,0 +1,1151 @@ +//! ID5 identity provider: resolves the ID5 ID server side. +//! +//! Posts the request's consent and network signals to ID5's fetch endpoint. +//! It needs no login and sends no hashed email. ID5 returns complete EIDs +//! under `ids`, or the legacy `universal_uid`, which is given to the first +//! configured source with the reply's `ext.linkType` as UID `ext`, as +//! Prebid's ID5 module does. +//! +//! Parsing is lenient per entry: an entry of `ids`, `universal_uid` or +//! `cache_control` with an unexpected shape is skipped rather than failing +//! the whole response. The opt-out UID `"0"` is never stored or forwarded. +//! `cache_control.max_age_sec` is bounded by [`bounded_expiry`]: a missing +//! value gets the default TTL and a zero or tiny one the minimum TTL. +//! +//! Before any lookup, the provider reads ID5's first-party cookies listed +//! in `cookie_names` (default `id5id`, written by Prebid's ID5 module), in +//! order, then, when the Lockr capability is enabled, the cookies Lockr AIM +//! writes (`[settings.lockr]`, default key `id5id`). A cookie holding the ID5 fetch response JSON +//! (plain or percent-encoded) is parsed exactly like a lookup response, and +//! a bare ID string becomes one UID on the first source with agent type 1 +//! and the default TTL. A reply's non-empty `signature` is kept as the +//! UIDs' private refresh state, due a tenth of their TTL before they +//! expire, and sent back as `s` in the next fetch so ID5 can return the +//! same ID, as Prebid's ID5 module does. A +//! percent-encoded JSON cookie is decoded and its entries, `ext` objects +//! included, are copied out of the decoded text. A cookie that yields no +//! usable UID is ignored and the lookup runs as usual. A cookie holding +//! ID5's opt-out (a bare `0`, or a reply whose `universal_uid` is `0`) +//! reports an opt-out: the lookup is skipped and stored ID5 UIDs are +//! removed. A fetch reply whose `universal_uid` is `0` reports the same +//! opt-out. +//! +//! The request follows ID5's server-side client (its Prebid Server module): +//! `gpp_sid` is sent comma-joined, the client IP goes in `ip` (IPv4) or +//! `ipv6`, `provider` names who operates the integration (`provider_name`, +//! default the publisher domain), `domain` is the publisher domain and +//! `origin` is `trusted-server`. The field names for the client IP and user +//! agent should be confirmed with ID5 for each partner account. + +use std::borrow::Cow; +use std::collections::BTreeMap; +use std::net::IpAddr; +use std::num::NonZeroUsize; +use std::time::Duration; + +use bytes::Bytes; +use chrono::{DateTime, SecondsFormat, Utc}; +use error_stack::{Report, ResultExt as _}; +use http::header::{CONTENT_TYPE, ORIGIN}; +use http::{HeaderMap, HeaderValue, Method}; +use serde::{Deserialize, Serialize}; +use trusted_server_core::ec::eid_types::{SourceDomain, UnixSeconds}; +use trusted_server_core::ec::provider::{ + CookieRead, EidLevelFields, EidShape, IdentityProvider, LookupInput, LookupOutcome, + ProviderCapabilities, ProviderCommon, ProviderConfigError, ProviderEid, ProviderError, + ProviderExpiry, ProviderKind, ProviderRefresh, ProviderRequest, ProviderResponse, ProviderUid, + RequestCookies, SourceCapability, SourceInput, +}; +use url::Url; + +use serde_json::value::RawValue; + +use crate::common::{ + CookieReader, LockrSettings, ReadsLockr, VendorEid, bounded_expiry, is_routable_ip, + is_usable_id, join_section_ids, percent_decoded, +}; + +/// Formats the request time as the RFC 3339 `ts` the `/gs/v2` fetch requires. +fn request_timestamp(now: UnixSeconds) -> Result> { + let now = i64::try_from(now.get()).change_context(ProviderError::Request)?; + DateTime::::from_timestamp(now, 0) + .map(|time| time.to_rfc3339_opts(SecondsFormat::Secs, true)) + .ok_or_else(|| Report::new(ProviderError::Request).attach("current time is out of range")) +} + +/// Names this integration to ID5 in `origin`, as ID5's Prebid Server module +/// sends `pbs-java`. +const INTEGRATION_ORIGIN: &str = "trusted-server"; + +/// The UID ID5 returns for a visitor who opted out. +const OPTED_OUT_UID: &str = "0"; + +/// Most EIDs accepted from one ID5 response. +const MAX_EIDS: NonZeroUsize = NonZeroUsize::new(4).expect("should be non-zero"); + +/// Most UIDs accepted per ID5 EID. +const MAX_UIDS_PER_EID: NonZeroUsize = NonZeroUsize::new(2).expect("should be non-zero"); + +/// The ID5 provider kind. +pub struct Id5; + +/// Settings for an `id5` provider. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Id5Settings { + /// The ID5 partner number assigned to the publisher. + pub partner_id: u32, + /// First-party cookies read in order before any lookup; the first + /// yielding a usable UID wins. + #[serde(default = "Id5Settings::default_cookie_names")] + pub cookie_names: Vec, + /// Who operates this integration, sent to ID5 as `provider`. Defaults + /// to the publisher domain. + #[serde(default)] + pub provider_name: Option, + /// The Lockr capability; see [`LockrSettings`]. + #[serde(default)] + pub lockr: LockrSettings, +} + +impl Id5Settings { + fn default_cookie_names() -> Vec { + vec!["id5id".to_owned()] + } +} + +/// A built ID5 provider. +#[derive(Debug)] +pub struct Id5Provider { + capabilities: ProviderCapabilities, + url: Url, + partner_id: u32, + provider_name: Option, + cookies: CookieReader, +} + +impl ReadsLockr for Id5 { + const LOCKR_COOKIE_NAMES: &'static [&'static str] = &["id5id"]; +} + +impl ProviderKind for Id5 { + const KIND: &'static str = "id5"; + type Settings = Id5Settings; + type Provider = Id5Provider; + + fn build( + common: &ProviderCommon<'_>, + settings: Id5Settings, + ) -> Result> { + if common.sources.is_empty() { + return Err(Report::new(ProviderConfigError::Invalid) + .attach("an id5 provider needs at least one source")); + } + let mut url = common + .endpoint + .ok_or_else(|| { + Report::new(ProviderConfigError::Invalid).attach("a id5 provider needs an endpoint") + })? + .clone(); + url.path_segments_mut() + .map_err(|()| { + Report::new(ProviderConfigError::Invalid).attach("endpoint cannot hold a path") + })? + .pop_if_empty() + .push(&format!("{}.json", settings.partner_id)); + Ok(Id5Provider { + capabilities: ProviderCapabilities { + sources: common + .sources + .iter() + .map(|source| SourceCapability { + source: source.clone(), + input: SourceInput::Anonymous, + }) + .collect(), + shape: EidShape { + max_eids: MAX_EIDS, + max_uids_per_eid: MAX_UIDS_PER_EID, + eid_level_fields: EidLevelFields::Preserved, + }, + }, + url, + partner_id: settings.partner_id, + provider_name: settings.provider_name, + cookies: CookieReader::with_lockr::( + common, + settings.cookie_names, + settings.lockr, + )?, + }) + } +} + +#[derive(Serialize)] +struct FetchBody<'a> { + partner: u32, + provider: &'a str, + domain: &'a str, + origin: &'static str, + gdpr: u8, + #[serde(skip_serializing_if = "Option::is_none")] + gdpr_consent: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + us_privacy: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + gpp_string: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + gpp_sid: Option, + ts: String, + #[serde(skip_serializing_if = "Option::is_none")] + ua: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + ip: Option, + #[serde(skip_serializing_if = "Option::is_none")] + ipv6: Option, + /// The previous reply's `signature`, which lets ID5 return the same ID. + #[serde(skip_serializing_if = "Option::is_none")] + s: Option<&'a str>, +} + +/// The top-level reply; each field is parsed separately so one bad shape +/// does not fail the rest. +#[derive(Deserialize)] +struct FetchReply<'body> { + #[serde(default, borrow)] + universal_uid: Option<&'body RawValue>, + #[serde(default, borrow)] + ids: Option<&'body RawValue>, + #[serde(default, borrow)] + cache_control: Option<&'body RawValue>, + #[serde(default, borrow)] + ext: Option<&'body RawValue>, + #[serde(default, borrow)] + signature: Option<&'body RawValue>, +} + +#[derive(Deserialize)] +struct ReplyExt<'body> { + #[serde(rename = "linkType", default, borrow)] + link_type: Option<&'body RawValue>, +} + +#[derive(Deserialize)] +struct Id5Id<'body> { + #[serde(default, borrow)] + eid: Option>, +} + +#[derive(Deserialize)] +struct CacheControl { + max_age_sec: Option, +} + +impl Id5Provider { + fn primary_source(&self) -> &SourceDomain { + &self.capabilities.sources[0].source + } + + /// Returns the configured source matching `source` after normalization + /// (case and trailing dot). + fn known_source(&self, source: &str) -> Option<&SourceDomain> { + let source = SourceDomain::try_from(source).ok()?; + self.capabilities + .sources + .iter() + .map(|known| &known.source) + .find(|known| **known == source) + } + + /// Parses an ID5 fetch reply into its kept EIDs. + fn parse_reply<'body>( + &self, + body: &'body [u8], + ) -> Result>, serde_json::Error> { + let reply: FetchReply<'body> = serde_json::from_slice(body)?; + let max_age = reply + .cache_control + .and_then(|raw| serde_json::from_str::(raw.get()).ok()) + .and_then(|cache| cache.max_age_sec) + .map(Duration::from_secs); + let expiry = bounded_expiry(max_age); + let signature = reply + .signature + .and_then(|raw| serde_json::from_str::>(raw.get()).ok()) + .filter(|signature| !signature.is_empty()); + let refresh = || { + signature.as_ref().map(|signature| ProviderRefresh { + state: signature.clone(), + from: refresh_from(expiry), + }) + }; + + let entries: BTreeMap = reply + .ids + .and_then(|raw| serde_json::from_str(raw.get()).ok()) + .unwrap_or_default(); + let mut eids: Vec> = entries + .into_values() + .filter_map(|raw| serde_json::from_str::>(raw.get()).ok()) + .filter_map(|id| id.eid) + .filter_map(|mut eid| { + let source = self.known_source(&eid.source)?; + eid.source = Cow::Owned(source.as_str().to_owned()); + eid.uids.retain(|uid| is_usable_uid(&uid.id)); + (!eid.uids.is_empty()).then(|| eid.into_provider_eid(expiry)) + }) + .collect(); + for uid in eids.iter_mut().flat_map(|eid| eid.uids.iter_mut()) { + uid.refresh = refresh(); + } + let primary = self.primary_source().as_str(); + let has_primary = eids.iter().any(|eid| eid.source == primary); + let universal_uid = reply + .universal_uid + .and_then(|raw| serde_json::from_str::>(raw.get()).ok()) + .filter(|uid| is_usable_uid(uid)); + if !has_primary && let Some(uid) = universal_uid { + let link_type = reply + .ext + .and_then(|raw| serde_json::from_str::>(raw.get()).ok()) + .and_then(|ext| ext.link_type) + .and_then(|link_type| { + RawValue::from_string(format!(r#"{{"linkType":{}}}"#, link_type.get())).ok() + }); + eids.push(ProviderEid { + source: Cow::Owned(primary.to_owned()), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids: vec![ProviderUid { + id: uid, + atype: Some(1), + ext: link_type.map(Cow::Owned), + expiry, + refresh: refresh(), + }], + }); + } + Ok(eids) + } + + /// Reads one ID5 cookie value: a fetch reply JSON or a bare ID. + fn cookie_eids<'cookie>(&self, value: &'cookie str) -> Option>> { + let eids = match percent_decoded(value) { + Cow::Borrowed(value) if value.trim_start().starts_with('{') => { + self.parse_reply(value.as_bytes()).ok()? + } + Cow::Owned(value) if value.trim_start().starts_with('{') => self + .parse_reply(value.as_bytes()) + .ok()? + .into_iter() + .map(owned) + .collect(), + value => { + let id = value.trim(); + if !is_usable_uid(id) || !is_usable_id(id) { + return None; + } + vec![ProviderEid { + source: Cow::Owned(self.primary_source().as_str().to_owned()), + inserter: None, + matcher: None, + mm: None, + ext: None, + uids: vec![ProviderUid { + id: Cow::Owned(id.to_owned()), + atype: Some(1), + ext: None, + expiry: bounded_expiry(None), + refresh: None, + }], + }] + } + }; + (!eids.is_empty()).then_some(eids) + } +} + +impl IdentityProvider for Id5Provider { + fn capabilities(&self) -> &ProviderCapabilities { + &self.capabilities + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + let consent = lookup.consent; + let origin = format!("https://{}", lookup.publisher_domain); + let routable_ip = lookup.client.ip.filter(|ip| is_routable_ip(*ip)); + let body = FetchBody { + partner: self.partner_id, + provider: self + .provider_name + .as_deref() + .unwrap_or(lookup.publisher_domain), + domain: lookup.publisher_domain, + origin: INTEGRATION_ORIGIN, + gdpr: u8::from(consent.gdpr_applies), + gdpr_consent: consent.raw_tc_string.as_deref(), + us_privacy: consent.raw_us_privacy.as_deref(), + gpp_string: consent.raw_gpp_string.as_deref(), + gpp_sid: consent.gpp_section_ids.as_deref().map(join_section_ids), + ts: request_timestamp(lookup.now)?, + ua: lookup.client.user_agent, + ip: routable_ip.filter(IpAddr::is_ipv4).map(|ip| ip.to_string()), + ipv6: routable_ip.filter(IpAddr::is_ipv6).map(|ip| ip.to_string()), + s: lookup + .current + .iter() + .flat_map(|(_, eid)| eid.uids()) + .find_map(|uid| uid.refresh()) + .map(|refresh| refresh.state.as_str()), + }; + let body = serde_json::to_vec(&body).change_context(ProviderError::Request)?; + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + headers.insert( + ORIGIN, + HeaderValue::from_str(&origin).change_context(ProviderError::Request)?, + ); + Ok(ProviderRequest { + method: Method::POST, + url: self.url.clone(), + headers, + body: Bytes::from(body), + context: None, + }) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + if response.status == http::StatusCode::NO_CONTENT { + return Ok(LookupOutcome::NoMatch); + } + if !response.status.is_success() { + return Err(Report::new(ProviderError::InvalidResponse) + .attach(format!("ID5 returned status {}", response.status))); + } + if is_opt_out_reply(response.body) { + return Ok(LookupOutcome::OptedOut); + } + let eids = self + .parse_reply(response.body) + .change_context(ProviderError::InvalidResponse)?; + Ok(if eids.is_empty() { + LookupOutcome::NoMatch + } else { + LookupOutcome::Found(eids) + }) + } + + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> CookieRead<'cookie> { + if self.cookies.any(cookies, is_opt_out_cookie) { + return CookieRead::OptedOut; + } + CookieRead::from(self.cookies.read(cookies, |value| self.cookie_eids(value))) + } +} + +/// Whether one ID5 cookie value records an opt-out: a bare `0`, or a +/// (possibly percent-encoded) reply whose `universal_uid` is `0`. +fn is_opt_out_cookie(value: &str) -> bool { + let decoded = percent_decoded(value); + let decoded = decoded.trim(); + if !decoded.starts_with('{') { + return decoded == OPTED_OUT_UID; + } + is_opt_out_reply(decoded.as_bytes()) +} + +/// Whether a fetch reply body, taken as is, has a `universal_uid` of `0`. +fn is_opt_out_reply(body: &[u8]) -> bool { + serde_json::from_slice::(body) + .ok() + .and_then(|reply| { + reply + .get("universal_uid") + .and_then(serde_json::Value::as_str) + .map(|uid| uid == OPTED_OUT_UID) + }) + .unwrap_or(false) +} + +/// Copies an EID parsed from a temporary string so it outlives that string. +fn owned<'any>(eid: ProviderEid<'_>) -> ProviderEid<'any> { + ProviderEid { + source: Cow::Owned(eid.source.into_owned()), + inserter: eid.inserter.map(|value| Cow::Owned(value.into_owned())), + matcher: eid.matcher.map(|value| Cow::Owned(value.into_owned())), + mm: eid.mm, + ext: eid.ext.map(|ext| Cow::Owned(ext.into_owned())), + uids: eid + .uids + .into_iter() + .map(|uid| ProviderUid { + id: Cow::Owned(uid.id.into_owned()), + atype: uid.atype, + ext: uid.ext.map(|ext| Cow::Owned(ext.into_owned())), + expiry: uid.expiry, + refresh: uid.refresh.map(|refresh| ProviderRefresh { + state: Cow::Owned(refresh.state.into_owned()), + from: refresh.from, + }), + }) + .collect(), + } +} + +/// When a UID that expires at `expiry` is refreshed: a tenth of its TTL +/// early, so the refresh runs while the UID and its signature are still +/// stored. +fn refresh_from(expiry: ProviderExpiry) -> ProviderExpiry { + match expiry { + ProviderExpiry::After(ttl) => ProviderExpiry::After(ttl - ttl / 10), + other => other, + } +} + +/// Returns whether `id` is a real UID: not empty and not the opt-out UID. +fn is_usable_uid(id: &str) -> bool { + !id.is_empty() && id != OPTED_OUT_UID +} + +#[cfg(test)] +mod tests { + use http::StatusCode; + use serde_json::json; + use trusted_server_core::consent::types::ConsentContext; + use trusted_server_core::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, UidValue}; + use trusted_server_core::ec::kv_types::{KvEid, KvEntry, KvPartnerId, KvRefresh, KvUid}; + use trusted_server_core::ec::provider::{ + ClientSignals, OwnedEntries, ProviderExpiry, build_kind, + }; + use trusted_server_core::settings::{IdentityProviderConfig, ProviderSecrets}; + + use super::*; + + fn provider() -> Box { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "id5", + "kind": "id5", + "endpoint": "https://id5.example.com/gs/v2", + "allowed_domains": ["id5.example.com"], + "sources": ["id5.example.com", "id5-link.example.com"], + "settings": { "partner_id": 1234 }, + })) + .expect("should parse provider config"); + build_kind::(&config, &ProviderSecrets::default()).expect("should build provider") + } + + fn respond<'body>( + provider: &dyn IdentityProvider, + headers: &'body HeaderMap, + body: &'body [u8], + ) -> LookupOutcome<'body> { + provider + .parse_response(ProviderResponse { + status: StatusCode::OK, + headers, + body, + context: None, + }) + .expect("should parse response") + } + + #[test] + fn keeps_the_reply_signature_as_refresh_state() { + let provider = provider(); + let headers = HeaderMap::new(); + let body = json!({ + "universal_uid": "id5-example", + "signature": "signature-example", + "cache_control": { "max_age_sec": 36_000 }, + }) + .to_string(); + + let outcome = respond(&*provider, &headers, body.as_bytes()); + + let LookupOutcome::Found(eids) = outcome else { + panic!("should find the ID5 ID"); + }; + let refresh = eids[0].uids[0] + .refresh + .as_ref() + .expect("should keep the signature as refresh state"); + assert_eq!( + refresh.state, "signature-example", + "should store the signature" + ); + assert!( + matches!(refresh.from, ProviderExpiry::After(ttl) if ttl == Duration::from_secs(32_400)), + "should refresh a tenth of the TTL before the UID expires" + ); + } + + #[test] + fn keeps_no_refresh_state_without_a_signature() { + let provider = provider(); + let headers = HeaderMap::new(); + let body = json!({ "universal_uid": "id5-example", "signature": "" }).to_string(); + + let outcome = respond(&*provider, &headers, body.as_bytes()); + + let LookupOutcome::Found(eids) = outcome else { + panic!("should find the ID5 ID"); + }; + assert!( + eids[0].uids[0].refresh.is_none(), + "should keep no refresh state for an empty signature" + ); + } + + #[test] + fn sends_the_stored_signature_as_s() { + // Arrange + let uid = KvUid::new( + UidValue::try_from("id5-example".to_owned()).expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("id5")), + Expiry::At(UnixSeconds::new(1_800_000_000)), + ) + .with_refresh(KvRefresh { + state: "signature-example".to_owned(), + from: UnixSeconds::new(1_700_000_000), + }); + let record = KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![uid])], + UnixSeconds::new(1_700_000_000), + ) + .expect("should keep the live uid"); + let mut entry = KvEntry::tombstone(1); + entry.consent.ok = true; + entry.ids.insert("id5.example.com".to_owned(), record); + let sources = [SourceDomain::try_from("id5.example.com").expect("should build source")]; + let owner = ProviderId::new("id5"); + let consent = ConsentContext::default(); + let lookup = LookupInput { + ec_id: "ec-example", + consent: &consent, + client: ClientSignals { + ip: None, + user_agent: None, + }, + publisher_domain: "www.example.com", + due_sources: &sources, + current: OwnedEntries::new(Some(&entry), &sources, &owner), + cookies: RequestCookies::default(), + hashed_email: None, + now: UnixSeconds::new(1_700_000_000), + }; + + // Act + let request = provider() + .build_request(&lookup) + .expect("should build request"); + + // Assert + let body: serde_json::Value = + serde_json::from_slice(&request.body).expect("should send JSON"); + assert_eq!( + body.get("s"), + Some(&json!("signature-example")), + "should send the previous signature as s" + ); + } + + #[test] + fn posts_consent_and_network_signals_to_the_partner_url() { + let provider = provider(); + let consent = ConsentContext { + raw_us_privacy: Some("1YNN".to_owned()), + gpp_section_ids: Some(vec![7, 8]), + ..ConsentContext::default() + }; + let lookup = LookupInput { + ec_id: "ec-example", + consent: &consent, + client: ClientSignals { + ip: Some("11.22.33.44".parse().expect("should parse ip")), + user_agent: Some("ExampleBrowser/1.0"), + }, + publisher_domain: "www.example.com", + due_sources: &[], + current: OwnedEntries::empty(), + cookies: RequestCookies::default(), + hashed_email: None, + now: trusted_server_core::ec::eid_types::UnixSeconds::new(1_700_000_000), + }; + + let request = provider + .build_request(&lookup) + .expect("should build request"); + let body: serde_json::Value = + serde_json::from_slice(&request.body).expect("should send JSON"); + + assert_eq!(request.method, Method::POST, "should post"); + assert_eq!( + request.url.as_str(), + "https://id5.example.com/gs/v2/1234.json", + "should call the partner's fetch URL" + ); + assert_eq!( + request.headers.get(ORIGIN).map(HeaderValue::as_bytes), + Some(&b"https://www.example.com"[..]), + "should send the publisher origin" + ); + assert_eq!(body["partner"], 1234, "should send the partner"); + assert_eq!( + body["provider"], "www.example.com", + "should name the publisher as operator by default" + ); + assert_eq!( + body["domain"], "www.example.com", + "should send the publisher domain" + ); + assert_eq!( + body["origin"], "trusted-server", + "should name the integration" + ); + for legacy in ["o", "v", "tml", "nbPage"] { + assert!(body.get(legacy).is_none(), "should not send `{legacy}`"); + } + let ts = body["ts"].as_str().expect("should send a ts string"); + DateTime::parse_from_rfc3339(ts).expect("should send ts as an RFC 3339 timestamp"); + assert_eq!(body["gdpr"], 0, "should send that GDPR does not apply"); + assert_eq!(body["us_privacy"], "1YNN", "should send US privacy"); + assert_eq!( + body["gpp_sid"], "7,8", + "should send GPP section IDs comma-joined" + ); + assert_eq!(body["ip"], "11.22.33.44", "should send the client IP"); + assert_eq!( + body["ua"], "ExampleBrowser/1.0", + "should send the user agent" + ); + } + + #[test] + fn drops_a_private_client_ip() { + let provider = provider(); + let consent = ConsentContext::default(); + let lookup = LookupInput { + ec_id: "ec-example", + consent: &consent, + client: ClientSignals { + ip: Some("10.1.2.3".parse().expect("should parse ip")), + user_agent: None, + }, + publisher_domain: "www.example.com", + due_sources: &[], + current: OwnedEntries::empty(), + cookies: RequestCookies::default(), + hashed_email: None, + now: trusted_server_core::ec::eid_types::UnixSeconds::new(1_700_000_000), + }; + + let request = provider + .build_request(&lookup) + .expect("should build request"); + let body: serde_json::Value = + serde_json::from_slice(&request.body).expect("should send JSON"); + + assert!(body.get("ip").is_none(), "should not send a private IP"); + } + + #[test] + fn sends_an_ipv6_address_as_ipv6() { + let provider = provider(); + let consent = ConsentContext::default(); + let lookup = LookupInput { + ec_id: "ec-example", + consent: &consent, + client: ClientSignals { + ip: Some("2001:4860::1".parse().expect("should parse ip")), + user_agent: None, + }, + publisher_domain: "www.example.com", + due_sources: &[], + current: OwnedEntries::empty(), + cookies: RequestCookies::default(), + hashed_email: None, + now: trusted_server_core::ec::eid_types::UnixSeconds::new(1_700_000_000), + }; + + let request = provider + .build_request(&lookup) + .expect("should build request"); + let body: serde_json::Value = + serde_json::from_slice(&request.body).expect("should send JSON"); + + assert!(body.get("ip").is_none(), "should not send IPv6 as ip"); + assert_eq!(body["ipv6"], "2001:4860::1", "should send ipv6"); + } + + #[test] + fn parses_complete_eids_for_configured_sources() { + let provider = provider(); + let headers = HeaderMap::new(); + let body = serde_json::to_vec(&json!({ + "ids": { + "id5": { "eid": { "source": "id5.example.com", "uids": [{ "id": "id5-example", "atype": 1, "ext": { "linkType": 2 } }] } }, + "other": { "eid": { "source": "unknown.example.com", "uids": [{ "id": "x" }] } }, + }, + "cache_control": { "max_age_sec": 3600 }, + })) + .expect("should encode body"); + + let LookupOutcome::Found(eids) = respond(&*provider, &headers, &body) else { + panic!("should find an EID"); + }; + + assert_eq!(eids.len(), 1, "should drop unconfigured sources"); + assert_eq!(eids[0].source, "id5.example.com", "should keep the source"); + assert_eq!(eids[0].uids[0].id, "id5-example", "should keep the UID"); + assert!(eids[0].uids[0].ext.is_some(), "should keep the UID ext"); + assert!( + matches!(eids[0].uids[0].expiry, ProviderExpiry::After(d) if d == Duration::from_secs(3600)), + "should use the cache max age" + ); + } + + #[test] + fn falls_back_to_the_legacy_universal_uid() { + let provider = provider(); + let headers = HeaderMap::new(); + let body = serde_json::to_vec(&json!({ + "universal_uid": "legacy-example", + "ext": { "linkType": 2 }, + "ids": { + "id5": { "eid": { "source": "id5.example.com", "uids": [{ "id": "0" }] } }, + "broken": { "eid": "not an object" }, + }, + "cache_control": "not an object", + })) + .expect("should encode body"); + + let LookupOutcome::Found(eids) = respond(&*provider, &headers, &body) else { + panic!("should find an EID"); + }; + + assert_eq!( + eids[0].source, "id5.example.com", + "should use the first source" + ); + assert_eq!(eids.len(), 1, "should drop the opted-out EID"); + assert_eq!(eids[0].uids[0].id, "legacy-example", "should keep the UID"); + assert_eq!( + eids[0].uids[0].ext.as_deref().map(RawValue::get), + Some(r#"{"linkType":2}"#), + "should carry the reply linkType as UID ext" + ); + assert!( + matches!(eids[0].uids[0].expiry, ProviderExpiry::After(d) if d == crate::common::DEFAULT_TTL), + "should use the default TTL without a cache max age" + ); + } + + #[test] + fn normalizes_sources_skips_bad_entries_and_bounds_expiry() { + let provider = provider(); + let headers = HeaderMap::new(); + let body = serde_json::to_vec(&json!({ + "ids": { + "a": { "eid": { "source": "ID5.Example.com.", "uids": [{ "id": "id5-example" }, { "id": "0" }] } }, + "b": { "eid": { "source": 7 } }, + "c": { "eid": { "source": "id5-link.example.com", "uids": [{ "id": "0" }] } }, + }, + "cache_control": { "max_age_sec": 0 }, + })) + .expect("should encode body"); + + let LookupOutcome::Found(eids) = respond(&*provider, &headers, &body) else { + panic!("should find an EID"); + }; + + assert_eq!(eids.len(), 1, "should skip bad and opted-out entries"); + assert_eq!( + eids[0].source, "id5.example.com", + "should store the configured source" + ); + assert_eq!(eids[0].uids.len(), 1, "should drop the opt-out UID"); + assert_eq!(eids[0].uids[0].id, "id5-example", "should keep the UID"); + assert!( + matches!(eids[0].uids[0].expiry, ProviderExpiry::After(d) if d == crate::common::MIN_TTL), + "should keep a zero max age for the minimum TTL" + ); + } + + #[test] + fn reports_no_match_for_an_empty_reply() { + let provider = provider(); + let headers = HeaderMap::new(); + let body = serde_json::to_vec(&json!({})).expect("should encode body"); + + assert!( + matches!(respond(&*provider, &headers, &body), LookupOutcome::NoMatch), + "should report no match" + ); + } + + #[test] + fn reports_an_opt_out_reply() { + let provider = provider(); + let headers = HeaderMap::new(); + let body = serde_json::to_vec(&json!({ + "universal_uid": "0", + "cache_control": { "max_age_sec": 7200 }, + })) + .expect("should encode body"); + + let outcome = respond(&*provider, &headers, &body); + + assert!( + matches!(outcome, LookupOutcome::OptedOut), + "should report an opt-out for a universal_uid of 0" + ); + } + + #[test] + fn does_not_percent_decode_an_opt_out_reply() { + let provider = provider(); + let headers = HeaderMap::new(); + let body = serde_json::to_vec(&json!({ + "universal_uid": "0", + "ext": { "note": "a%22b" }, + })) + .expect("should encode body"); + + let outcome = respond(&*provider, &headers, &body); + + assert!( + matches!(outcome, LookupOutcome::OptedOut), + "should read the reply as sent" + ); + } + + #[test] + fn rejects_an_error_status_and_a_malformed_body() { + let provider = provider(); + let headers = HeaderMap::new(); + + for (status, body) in [ + (StatusCode::INTERNAL_SERVER_ERROR, &b"{}"[..]), + (StatusCode::OK, b"not json"), + ] { + let error = provider + .parse_response(ProviderResponse { + status, + headers: &headers, + body, + context: None, + }) + .expect_err("should reject the response"); + assert!( + matches!(error.current_context(), ProviderError::InvalidResponse), + "should report an invalid response" + ); + } + } + + /// Source, UID, agent type, whether a UID `ext` is kept, and TTL. + type ReadUid = (String, String, Option, bool, Duration); + + fn read(provider: &dyn IdentityProvider, header: &str) -> Option> { + provider + .read_request_cookies(RequestCookies::new(header)) + .into_found() + .map(|eids| { + eids.into_iter() + .flat_map(|eid| { + let source = eid.source.into_owned(); + eid.uids.into_iter().map(move |uid| { + let ProviderExpiry::After(ttl) = uid.expiry else { + panic!("should state a relative expiry"); + }; + ( + source.clone(), + uid.id.into_owned(), + uid.atype, + uid.ext.is_some(), + ttl, + ) + }) + }) + .collect() + }) + } + + fn reply_cookie() -> serde_json::Value { + json!({ + "universal_uid": "legacy-example", + "signature": "signature-example", + "ids": { + "id5": { "eid": { "source": "id5.example.com", "uids": [{ "id": "id5-example", "atype": 1, "ext": { "linkType": 2 } }] } }, + }, + "cache_control": { "max_age_sec": 7200 }, + }) + } + + #[test] + fn reads_a_plain_json_reply_cookie() { + let provider = provider(); + let header = format!("other=1; id5id={}", reply_cookie()); + + let uids = read(&*provider, &header).expect("should read the cookie"); + + assert_eq!( + uids, + vec![( + "id5.example.com".to_owned(), + "id5-example".to_owned(), + Some(1), + true, + Duration::from_secs(7200) + )], + "should parse the reply like a lookup response" + ); + } + + #[test] + fn reads_a_percent_encoded_json_reply_cookie() { + let provider = provider(); + let header = format!("id5id={}", urlencoding::encode(&reply_cookie().to_string())); + + let uids = read(&*provider, &header).expect("should read the cookie"); + + assert_eq!( + uids, + vec![( + "id5.example.com".to_owned(), + "id5-example".to_owned(), + Some(1), + true, + Duration::from_secs(7200) + )], + "should decode the reply and keep ext objects" + ); + } + + #[test] + fn reads_a_bare_id_cookie_from_a_configured_name() { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "id5", + "kind": "id5", + "endpoint": "https://id5.example.com/gs/v2", + "allowed_domains": ["id5.example.com"], + "sources": ["id5.example.com"], + "settings": { "partner_id": 1234, "cookie_names": ["example_lockr_id5", "id5id"] }, + })) + .expect("should parse provider config"); + let provider = + build_kind::(&config, &ProviderSecrets::default()).expect("should build provider"); + + let uids = read(&provider, "id5id=%7Bbroken; example_lockr_id5=bare-example") + .expect("should read the bare ID"); + + assert_eq!( + uids, + vec![( + "id5.example.com".to_owned(), + "bare-example".to_owned(), + Some(1), + false, + crate::common::DEFAULT_TTL + )], + "should store the bare ID on the first source with the default TTL" + ); + } + + #[test] + fn reads_lockr_after_own_cookies_and_falls_back_to_the_lookup() { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "id5", + "kind": "id5", + "endpoint": "https://id5.example.com/gs/v2", + "allowed_domains": ["id5.example.com"], + "sources": ["id5.example.com"], + "settings": { + "partner_id": 1234, + "lockr": { "enabled": true, "cookie_names": ["example_lockr_id5"] }, + }, + })) + .expect("should parse provider config"); + let provider = + build_kind::(&config, &ProviderSecrets::default()).expect("should build provider"); + + let own_first = read( + &provider, + "example_lockr_id5=lockr-example; id5id=own-example", + ); + let lockr = read( + &provider, + "id5id=%7Bbroken; example_lockr_id5=lockr-example", + ); + let neither = read(&provider, "example_lockr_id5=%7Bbroken"); + + assert_eq!( + own_first.map(|uids| uids[0].1.clone()).as_deref(), + Some("own-example"), + "should prefer the own cookie" + ); + assert_eq!( + lockr.map(|uids| uids[0].1.clone()).as_deref(), + Some("lockr-example"), + "should read the Lockr cookie when the own cookie yields nothing" + ); + assert!(neither.is_none(), "should leave the ID to the lookup"); + assert!( + provider + .read_request_cookies( + RequestCookies::new("example_lockr_id5=lockr-example").where_gdpr_applies(true) + ) + .into_found() + .is_none(), + "should not read Lockr cookies where GDPR applies, as Lockr AIM does not run there" + ); + } + + #[test] + fn reports_an_opt_out_cookie_so_the_lookup_is_skipped() { + let provider = provider(); + let opted_out = json!({ "universal_uid": "0", "ids": {} }).to_string(); + + for header in [ + "id5id=0".to_owned(), + format!("id5id={opted_out}"), + format!("id5id={}", urlencoding::encode(&opted_out)), + ] { + assert!( + matches!( + provider.read_request_cookies(RequestCookies::new(&header)), + CookieRead::OptedOut + ), + "should report an opt-out from {header}" + ); + } + } + + #[test] + fn ignores_malformed_cookies_so_the_lookup_runs() { + let provider = provider(); + + for header in ["id5id={not json", "other=bare-example"] { + assert!( + matches!( + provider.read_request_cookies(RequestCookies::new(header)), + CookieRead::Absent + ), + "should read nothing from {header}" + ); + } + } +} diff --git a/crates/trusted-server-identity-providers/src/lib.rs b/crates/trusted-server-identity-providers/src/lib.rs index e11e69abf..1a2abc79a 100644 --- a/crates/trusted-server-identity-providers/src/lib.rs +++ b/crates/trusted-server-identity-providers/src/lib.rs @@ -9,52 +9,119 @@ //! Dependencies point one way: this crate depends on core, and core never //! depends on it. +mod common; + +pub use common::LockrSettings; +pub mod connectid; +pub mod criteo; +pub mod id5; +pub mod liveintent; +pub mod liveramp; +pub mod sharedid; +pub mod uid2; +pub mod unifiedid; + +use std::borrow::Cow; use std::sync::Arc; use std::time::Duration; use error_stack::Report; +use trusted_server_core::config::with_placeholder_partner_tokens; use trusted_server_core::ec::coordinator::{ProviderCoordinator, RegisteredProvider}; -use trusted_server_core::ec::provider::{IdentityProvider, ProviderConfigError}; +use trusted_server_core::ec::provider::{ + IdentityProvider, ProviderConfigError, ProviderKind as _, build_kind, +}; use trusted_server_core::ec::registry::PartnerRegistry; use trusted_server_core::error::TrustedServerError; +use trusted_server_core::integrations::prebid::{PrebidIntegrationConfig, PrebidUserIdStorageType}; use trusted_server_core::settings::{ CollectTiming, Ec, IdentityProviderConfig, ProviderSecrets, Settings, }; /// Whether `kind` has a registered implementation. /// -/// No kinds are registered yet. #[must_use] -pub fn is_registered_kind(_kind: &str) -> bool { - false +pub fn is_registered_kind(kind: &str) -> bool { + matches!( + kind, + id5::Id5::KIND + | liveintent::LiveIntent::KIND + | liveramp::LiveRamp::KIND + | connectid::ConnectId::KIND + | uid2::Uid2::KIND + | sharedid::SharedId::KIND + | criteo::Criteo::KIND + | unifiedid::UnifiedId::KIND + ) +} + +/// Whether `kind` declares the Lockr capability, so its settings accept a +/// `[ec.identity_providers.settings.lockr]` table. +/// +/// The settings of every other kind reject that table as an unknown field +/// when the provider is built. +#[must_use] +pub fn reads_lockr(kind: &str) -> bool { + matches!( + kind, + id5::Id5::KIND | liveramp::LiveRamp::KIND | uid2::Uid2::KIND + ) +} + +/// Whether a provider of `kind` excludes `source` where GDPR applies. +fn excludes_where_gdpr_applies(kind: &str, source: &str) -> bool { + match kind { + uid2::Uid2::KIND => uid2::excludes_where_gdpr_applies(source), + liveintent::LiveIntent::KIND => liveintent::excludes_where_gdpr_applies(source), + _ => false, + } } /// Builds the partner registry, marking the sources claimed by enabled /// providers as provider-owned. /// +/// The sources any configured provider, enabled or disabled, excludes where +/// GDPR applies are marked too, so the UIDs already stored for them are +/// withheld from egress for a GDPR visitor. +/// /// # Errors /// /// Returns the same errors as [`PartnerRegistry::from_ec_with_kinds`], /// including for an enabled provider whose kind is not registered. pub fn partner_registry(ec: &Ec) -> Result> { - PartnerRegistry::from_ec_with_kinds(ec, is_registered_kind) + let mut registry = PartnerRegistry::from_ec_with_kinds(ec, is_registered_kind)?; + for provider in &ec.identity_providers { + for source in &provider.sources { + if excludes_where_gdpr_applies(provider.kind(), source) { + registry.exclude_where_gdpr_applies(source); + } + } + } + Ok(registry) } /// Builds the provider for a configured `kind`. /// -/// No kinds are registered yet, so every kind is rejected. -/// /// # Errors /// /// Returns [`ProviderConfigError::UnknownKind`] for an unregistered kind, or /// any error from [`trusted_server_core::ec::provider::build_kind`]. pub fn build( config: &IdentityProviderConfig, - _secrets: &ProviderSecrets, + secrets: &ProviderSecrets, ) -> Result, Report> { - let kind = config.kind(); - Err(Report::new(ProviderConfigError::UnknownKind) - .attach(format!("kind `{kind}` is not registered"))) + match config.kind() { + id5::Id5::KIND => build_kind::(config, secrets), + liveintent::LiveIntent::KIND => build_kind::(config, secrets), + liveramp::LiveRamp::KIND => build_kind::(config, secrets), + connectid::ConnectId::KIND => build_kind::(config, secrets), + uid2::Uid2::KIND => build_kind::(config, secrets), + sharedid::SharedId::KIND => build_kind::(config, secrets), + criteo::Criteo::KIND => build_kind::(config, secrets), + unifiedid::UnifiedId::KIND => build_kind::(config, secrets), + kind => Err(Report::new(ProviderConfigError::UnknownKind) + .attach(format!("kind `{kind}` is not registered"))), + } } /// Builds the coordinator for enabled providers collected before the @@ -64,9 +131,8 @@ pub fn build( /// /// # Errors /// -/// Returns [`ProviderConfigError`] when a provider cannot be built or the -/// enabled providers, taken together, fail the coordinator's configuration -/// checks. +/// Returns [`ProviderConfigError`] when a provider cannot be built or fails +/// the coordinator's configuration checks. pub fn build_coordinator( settings: &Settings, ) -> Result, Report> { @@ -116,6 +182,91 @@ pub fn build_after_response_coordinator( ) } +/// Builds every enabled provider of both collection phases, as the adapters +/// do on each navigation, so a provider-specific setting error is reported +/// before deploy. +/// +/// At runtime such an error disables every provider for the request, so +/// deploy-time config validation (`ts config push`, `validate` and `diff`) +/// calls this. Partner tokens may still be secret key names at that point; +/// they are replaced with placeholders for the check. +/// +/// # Errors +/// +/// Returns [`ProviderConfigError`] for the first provider that cannot be +/// built or fails the coordinator's configuration checks. +pub fn validate_providers(settings: &Settings) -> Result<(), Report> { + let ec = with_placeholder_partner_tokens(&settings.ec); + let partners = Arc::new( + partner_registry(&ec) + .map_err(|report| report.change_context(ProviderConfigError::Invalid))?, + ); + build_phase_coordinator( + settings, + Arc::clone(&partners), + CollectTiming::Page, + Duration::from_millis(settings.ec.identity_budget_ms), + )?; + build_phase_coordinator( + settings, + partners, + CollectTiming::AfterResponse, + Duration::from_millis(settings.ec.identity_after_response_budget_ms), + )?; + Ok(()) +} + +/// Prebid `userSync.userIds` names of the `SharedID` module. +const PREBID_SHAREDID_NAMES: [&str; 2] = ["sharedId", "pubCommonId"]; + +/// The cookie Prebid's `SharedID` module stores its ID in, as configured in +/// `integrations.prebid.managed_user_ids[].storage.name`. +/// +/// `None` when Prebid is not configured, manages no `SharedID` module, or +/// keeps it in local storage. +fn prebid_sharedid_cookie(settings: &Settings) -> Option { + let prebid = match settings.integration_config::("prebid") { + Ok(prebid) => prebid?, + Err(report) => { + log::warn!("Identity providers: Prebid config unreadable: {report}"); + return None; + } + }; + prebid + .managed_user_ids + .into_iter() + .filter(|entry| { + PREBID_SHAREDID_NAMES + .iter() + .any(|name| entry.name.eq_ignore_ascii_case(name)) + }) + .find_map(|entry| entry.storage) + .filter(|storage| storage.storage_type == PrebidUserIdStorageType::Cookie) + .map(|storage| storage.name) +} + +/// Fills a `sharedid` provider's `cookie_names` with the cookie Prebid's +/// `SharedID` module is configured to use, unless the provider sets its own. +fn with_prebid_sharedid_cookie<'config>( + config: &'config IdentityProviderConfig, + settings: &Settings, +) -> Cow<'config, IdentityProviderConfig> { + if config.kind() != sharedid::SharedId::KIND + || config.settings.contains_key(sharedid::COOKIE_NAMES_SETTING) + { + return Cow::Borrowed(config); + } + let Some(cookie) = prebid_sharedid_cookie(settings) else { + return Cow::Borrowed(config); + }; + let mut config = config.clone(); + config.settings.insert( + sharedid::COOKIE_NAMES_SETTING.to_owned(), + serde_json::Value::from(vec![cookie]), + ); + Cow::Owned(config) +} + fn build_phase_coordinator( settings: &Settings, partners: Arc, @@ -131,8 +282,9 @@ fn build_phase_coordinator( .iter() .filter(|config| config.enabled && config.collect == timing) .map(|config| { - build(config, &config.secrets) - .map(|provider| RegisteredProvider::new(config.clone(), provider)) + let config = with_prebid_sharedid_cookie(config, settings); + build(&config, &config.secrets) + .map(|provider| RegisteredProvider::new(config.into_owned(), provider)) }) .collect::, Report>>()?; if registered.is_empty() { @@ -143,7 +295,17 @@ fn build_phase_coordinator( #[cfg(test)] mod tests { + use serde_json::json; + use super::*; + use std::num::NonZeroU32; + + use trusted_server_core::ec::provider::validate_enabled_configs; + use trusted_server_core::settings::IdentityProviderConfig; + + fn partners_for(settings: &Settings) -> Arc { + Arc::new(partner_registry(&settings.ec).expect("should build partner registry")) + } fn create_test_settings() -> Settings { Settings::from_toml( @@ -166,6 +328,80 @@ mod tests { .expect("should parse test settings") } + #[test] + fn only_lockr_capable_kinds_accept_lockr_settings() { + for (kind, endpoint, sources, settings) in [ + ( + "id5", + Some("https://id5.example.com/gs/v2"), + json!(["id5.example.com"]), + json!({ "partner_id": 1234 }), + ), + ( + "liveintent", + Some("https://li.example.com/idex"), + json!(["liveintent.com"]), + json!({ "resolve": ["nonId"] }), + ), + ("liveramp", None, json!(["liveramp.example.com"]), json!({})), + ( + "connectid", + None, + json!(["connectid.example.com"]), + json!({}), + ), + ("uid2", None, json!(["uid2.example.com"]), json!({})), + ("sharedid", None, json!(["sharedid.example.com"]), json!({})), + ("criteo", None, json!(["criteo.example.com"]), json!({})), + ( + "unifiedid", + None, + json!(["unifiedid.example.com"]), + json!({}), + ), + ] { + let mut settings = settings; + settings["lockr"] = json!({ "enabled": true }); + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": kind, + "kind": kind, + "endpoint": endpoint, + "allowed_domains": ["id5.example.com", "li.example.com"], + "sources": sources, + "settings": settings, + })) + .expect("should parse provider config"); + + let built = build(&config, &ProviderSecrets::default()); + + assert_eq!( + built.is_ok(), + reads_lockr(kind), + "should accept lockr settings only for capable kind {kind}" + ); + } + } + + #[test] + fn enabled_lockr_rejects_an_empty_cookie_list() { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "uid2", + "kind": "uid2", + "sources": ["uid2.example.com"], + "settings": { "lockr": { "enabled": true, "cookie_names": [] } }, + })) + .expect("should parse provider config"); + + let error = build(&config, &ProviderSecrets::default()) + .err() + .expect("should reject an empty lockr cookie list"); + + assert!( + matches!(error.current_context(), ProviderConfigError::Invalid), + "should report invalid config" + ); + } + #[test] fn unknown_kind_is_rejected() { let config: IdentityProviderConfig = toml::from_str( @@ -219,36 +455,325 @@ mod tests { ); } + fn settings_with_cookie_providers(sharedid_collect: &str, criteo_collect: &str) -> Settings { + let mut settings = create_test_settings(); + settings.ec.partners = ["pubcid.org", "criteo.com"] + .iter() + .map(|source| { + toml::from_str(&format!( + "name = \"{source}\"\nsource_domain = \"{source}\"\nbidstream_enabled = true" + )) + .expect("should parse partner") + }) + .collect(); + settings.ec.identity_providers = vec![ + toml::from_str(&format!( + "id = \"sharedid\"\nkind = \"sharedid\"\nsources = [\"pubcid.org\"]\ncollect = \"{sharedid_collect}\"" + )) + .expect("should parse sharedid"), + toml::from_str(&format!( + "id = \"criteo\"\nkind = \"criteo\"\nsources = [\"criteo.com\"]\ncollect = \"{criteo_collect}\"" + )) + .expect("should parse criteo"), + ]; + settings + } + + fn with_prebid_sharedid(settings: &mut Settings, storage: &serde_json::Value) { + settings + .integrations + .insert_config( + "prebid", + &json!({ + "enabled": true, + "external_bundle_url": "https://assets.example/prebid/trusted-prebid.js", + "bundle": { "modules": { "bidder": ["exampleBidderBidAdapter"] } }, + "managed_user_ids": [{ "name": "sharedId", "storage": storage }], + }), + ) + .expect("should configure Prebid"); + } + #[test] - fn rejects_an_unknown_collect_timing() { - toml::from_str::( - "id = \"example-provider\"\nkind = \"example-kind\"\ncollect = \"later\"", + fn sharedid_reads_the_cookie_prebid_is_configured_to_use() { + // Arrange + let mut settings = settings_with_cookie_providers("page", "page"); + with_prebid_sharedid( + &mut settings, + &json!({ "type": "cookie", "name": "_pubcid" }), + ); + + // Act + let config = with_prebid_sharedid_cookie(&settings.ec.identity_providers[0], &settings); + + // Assert + assert_eq!( + config.settings.get("cookie_names"), + Some(&json!(["_pubcid"])), + "should read only Prebid's storage name" + ); + } + + #[test] + fn sharedid_keeps_its_defaults_without_a_prebid_cookie() { + let mut settings = settings_with_cookie_providers("page", "page"); + with_prebid_sharedid( + &mut settings, + &json!({ "type": "html5", "name": "_pubcid" }), + ); + + let config = with_prebid_sharedid_cookie(&settings.ec.identity_providers[0], &settings); + + assert!( + matches!(config, Cow::Borrowed(_)), + "should not take a local-storage key as a cookie name" + ); + } + + #[test] + fn sharedid_cookie_names_setting_wins_over_prebid() { + let mut settings = settings_with_cookie_providers("page", "page"); + with_prebid_sharedid( + &mut settings, + &json!({ "type": "cookie", "name": "_pubcid" }), + ); + settings.ec.identity_providers[0] + .settings + .insert("cookie_names".to_owned(), json!(["_example_sharedid"])); + + let config = with_prebid_sharedid_cookie(&settings.ec.identity_providers[0], &settings); + + assert_eq!( + config.settings.get("cookie_names"), + Some(&json!(["_example_sharedid"])), + "should keep the provider's own cookie names" + ); + } + + fn liveramp_provider(collect: &str, settings: &str) -> IdentityProviderConfig { + toml::from_str(&format!( + "id = \"liveramp\"\nkind = \"liveramp\"\nsources = [\"liveramp.example.com\"]\n\ + endpoint = \"https://envelope.example.com/api/identity/v2/envelope\"\n\ + allowed_domains = [\"envelope.example.com\"]\ncollect = \"{collect}\"\n\ + [settings]\n{settings}" + )) + .expect("should parse liveramp") + } + + fn settings_with_liveramp(collect: &str, settings: &str) -> Settings { + let mut configured = settings_with_cookie_providers("page", "page"); + configured.ec.partners.push( + toml::from_str( + "name = \"liveramp\"\nsource_domain = \"liveramp.example.com\"\nbidstream_enabled = true", + ) + .expect("should parse partner"), + ); + configured + .ec + .identity_providers + .push(liveramp_provider(collect, settings)); + configured + } + + #[test] + fn validate_providers_accepts_buildable_providers_in_both_phases() { + let page = settings_with_liveramp( + "page", + "placement_id = 14\ncustom_id_namespace = \"ana-example\"", + ); + let after = settings_with_liveramp( + "after_response", + "placement_id = 14\ncustom_id_namespace = \"ana-example\"", + ); + + validate_providers(&page).expect("should accept a buildable page provider"); + validate_providers(&after).expect("should accept a buildable after-response provider"); + } + + #[test] + fn validate_providers_rejects_a_provider_that_cannot_be_built() { + for collect in ["page", "after_response"] { + let settings = settings_with_liveramp(collect, "placement_id = 14"); + + assert!( + validate_providers(&settings).is_err(), + "should reject a {collect} liveramp endpoint without a namespace" + ); + } + } + + #[test] + fn validate_providers_rejects_an_enabled_unregistered_kind() { + let mut settings = settings_with_cookie_providers("page", "page"); + settings.ec.identity_providers.push( + toml::from_str("id = \"missing\"\nkind = \"missing\"\nsources = [\"criteo.com\"]") + .expect("should parse provider"), + ); + + let error = validate_providers(&settings) + .expect_err("should refuse an enabled provider whose kind is not registered"); + + assert!( + format!("{error:?}").contains("which this build does not include"), + "should name the unregistered kind: {error:?}" + ); + } + + #[test] + fn validate_providers_accepts_partner_tokens_that_are_still_key_names() { + let mut settings = settings_with_cookie_providers("page", "page"); + settings.ec.partners[0] = toml::from_str( + "name = \"pubcid.org\"\nsource_domain = \"pubcid.org\"\nbidstream_enabled = true\napi_token = \"partner_token_key\"", ) - .expect_err("should reject an unknown `collect` value"); + .expect("should parse partner with a token key name"); + + validate_providers(&settings).expect("should not resolve partner token key names"); } #[test] fn parses_an_optional_hourly_rate_limit() { let limited: IdentityProviderConfig = toml::from_str( - "id = \"example-provider\"\nkind = \"example-kind\"\nendpoint = \"https://ids.example.com/v1\"\nrate_limit_per_hour = 2", + "id = \"sharedid\"\nkind = \"sharedid\"\nsources = [\"pubcid.org\"]\nrate_limit_per_hour = 2", ) .expect("should parse a rate limit"); - let unset: IdentityProviderConfig = - toml::from_str("id = \"example-provider\"\nkind = \"example-kind\"\nendpoint = \"https://ids.example.com/v1\"") - .expect("should parse without a rate limit"); + let zero = toml::from_str::( + "id = \"sharedid\"\nkind = \"sharedid\"\nsources = [\"pubcid.org\"]\nrate_limit_per_hour = 0", + ); assert_eq!( - limited.rate_limit_per_hour.map(std::num::NonZeroU32::get), + limited.rate_limit_per_hour.map(NonZeroU32::get), Some(2), "should keep the hourly limit" ); + assert!(zero.is_err(), "should reject a zero limit"); + let settings = settings_with_cookie_providers("page", "page"); assert!( - unset.rate_limit_per_hour.is_none(), + settings.ec.identity_providers[0] + .rate_limit_per_hour + .is_none(), "should leave the limit unset by default" ); + } + + #[test] + fn providers_default_to_page_collection() { + let mut settings = settings_with_cookie_providers("page", "page"); + settings.ec.identity_providers = vec![ + toml::from_str("id = \"sharedid\"\nkind = \"sharedid\"\nsources = [\"pubcid.org\"]") + .expect("should parse sharedid"), + ]; + + let page = build_coordinator(&settings).expect("should build page coordinator"); + let after = build_after_response_coordinator(&settings, partners_for(&settings)) + .expect("should build after-response coordinator"); + + assert!( + page.is_some(), + "should collect a provider without `collect` on the page" + ); + assert!( + after.is_none(), + "should build no after-response coordinator" + ); + } + + #[test] + fn splits_providers_by_collect_timing() { + let settings = settings_with_cookie_providers("page", "after_response"); + + let page = build_coordinator(&settings) + .expect("should build page coordinator") + .expect("should have a page provider"); + let after = build_after_response_coordinator(&settings, partners_for(&settings)) + .expect("should build after-response coordinator") + .expect("should have an after-response provider"); + + assert!(!page.is_empty(), "should hold the page provider"); + assert!(!after.is_empty(), "should hold the after-response provider"); + } + + #[test] + fn all_after_response_builds_no_page_coordinator() { + let settings = settings_with_cookie_providers("after_response", "after_response"); + + let page = build_coordinator(&settings).expect("should build page coordinator"); + let after = build_after_response_coordinator(&settings, partners_for(&settings)) + .expect("should build after-response coordinator"); + + assert!(page.is_none(), "should build no page coordinator"); + assert!( + after.is_some(), + "should build the after-response coordinator" + ); + } + + #[test] + fn rejects_a_source_claimed_in_both_phases() { + let mut settings = settings_with_cookie_providers("page", "after_response"); + settings.ec.identity_providers[1] = toml::from_str( + "id = \"sharedid-late\"\nkind = \"sharedid\"\nsources = [\"pubcid.org\"]\ncollect = \"after_response\"", + ) + .expect("should parse second sharedid"); + + let partners = partner_registry(&settings.ec).expect("should build partner registry"); + + let result = validate_enabled_configs(&settings.ec.identity_providers, &partners); + + assert!( + result.is_err(), + "should reject a source claimed by providers in different phases" + ); + } + + #[test] + fn partner_registry_marks_sources_excluded_where_gdpr_applies() { + // Arrange: a disabled `uid2` provider still stored UIDs, and + // `liveintent` resolves UID2 next to its own source. + let mut settings = settings_with_cookie_providers("page", "page"); + for source in ["uidapi.com", "liveintent.example.com"] { + settings.ec.partners.push( + toml::from_str(&format!( + "name = \"{source}\"\nsource_domain = \"{source}\"\nbidstream_enabled = true" + )) + .expect("should parse partner"), + ); + } + settings.ec.identity_providers.push( + toml::from_str( + "id = \"uid2\"\nkind = \"uid2\"\nenabled = false\nsources = [\"uidapi.com\"]", + ) + .expect("should parse uid2"), + ); + settings.ec.identity_providers.push( + toml::from_str( + "id = \"liveintent\"\nkind = \"liveintent\"\nenabled = false\n\ + sources = [\"liveintent.example.com\", \"uidapi.com\"]", + ) + .expect("should parse liveintent"), + ); + + // Act + let registry = partner_registry(&settings.ec).expect("should build partner registry"); + + // Assert + let excluded = |source: &str| { + registry + .get(source) + .is_some_and(|partner| partner.excluded_where_gdpr_applies) + }; + assert!(excluded("uidapi.com"), "should mark the UID2 source"); + assert!( + !excluded("liveintent.example.com"), + "should not mark LiveIntent's own source" + ); + assert!(!excluded("pubcid.org"), "should not mark other sources"); + } + + #[test] + fn rejects_an_unknown_collect_timing() { toml::from_str::( - "id = \"example-provider\"\nkind = \"example-kind\"\nendpoint = \"https://ids.example.com/v1\"\nrate_limit_per_hour = 0", + "id = \"sharedid\"\nkind = \"sharedid\"\ncollect = \"later\"", ) - .expect_err("should reject a zero limit"); + .expect_err("should reject an unknown `collect` value"); } } diff --git a/crates/trusted-server-identity-providers/src/liveintent.rs b/crates/trusted-server-identity-providers/src/liveintent.rs new file mode 100644 index 000000000..bac4fd047 --- /dev/null +++ b/crates/trusted-server-identity-providers/src/liveintent.rs @@ -0,0 +1,1014 @@ +//! `LiveIntent` identity provider: resolves `LiveIntent`'s IDs server side. +//! +//! Calls `LiveIntent`'s identity exchange with the visitor's first-party +//! `LiveIntent` cookie (`_lc2_fpi` by default, sent as `duid`), consent and +//! network signals, and asks for the attributes the operator listed in +//! `resolve`. Each attribute maps to the `OpenRTB` source in +//! [`ATTRIBUTE_SOURCES`], which must be one of the provider's configured +//! sources. No hashed email is sent. +//! +//! As in Prebid's `LiveIntent` module, every attribute other than `nonId` is +//! a partner ID and its UID carries `ext.provider` naming `LiveIntent`. The +//! Trade Desk ID instead carries `ext.rtiPartner` and an `ext.provider` +//! naming the publisher's domain, and its EID names `LiveIntent` as +//! `matcher`. A non-empty `segments` array and a non-empty `m` value in the +//! reply go in the `nonId` EID's `ext`. +//! +//! This kind has no Lockr capability: Lockr AIM is not documented to +//! deliver `LiveIntent` IDs, so a `lockr` settings table is rejected. + +use std::borrow::Cow; +use std::collections::HashMap; +use std::num::NonZeroUsize; +use std::time::{Duration, SystemTime}; + +use base64::Engine as _; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use bytes::Bytes; +use error_stack::{Report, ResultExt as _}; +use http::header::{DATE, EXPIRES, ORIGIN}; +use http::{HeaderMap, HeaderValue, Method, StatusCode}; +use serde::Deserialize; +use serde_json::value::RawValue; +use trusted_server_core::ec::eid_types::{MAX_EXT_BYTES, SourceDomain}; +use trusted_server_core::ec::provider::{ + EidLevelFields, EidShape, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, + ProviderCommon, ProviderConfigError, ProviderEid, ProviderError, ProviderExpiry, ProviderKind, + ProviderRequest, ProviderResponse, ProviderUid, SourceCapability, SourceInput, +}; +use url::Url; + +use crate::common::{DEVICE_ATYPE, bounded_expiry, is_routable_ip, is_usable_id, join_section_ids}; + +/// Each attribute `LiveIntent` resolves and the source it carries. +pub const ATTRIBUTE_SOURCES: &[(&str, &str)] = &[ + (NON_ID_ATTRIBUTE, "liveintent.com"), + ("bidswitch", "bidswitch.net"), + ("medianet", "media.net"), + ("magnite", "rubiconproject.com"), + ("index", "liveintent.indexexchange.com"), + ("openx", "openx.net"), + ("pubmatic", "pubmatic.com"), + ("sovrn", "liveintent.sovrn.com"), + ("sharethrough", "sharethrough.com"), + ("sonobi", "liveintent.sonobi.com"), + ("triplelift", "liveintent.triplelift.com"), + ("zetassp", "zeta-ssp.liveintent.com"), + ("vidazoo", "liveintent.vidazoo.com"), + ("nexxen", "liveintent.unrulymedia.com"), + ("uid2", UID2_SOURCE), + (TDID_ATTRIBUTE, "adserver.org"), +]; + +/// The source of UID2 tokens `LiveIntent` resolves. +const UID2_SOURCE: &str = "uidapi.com"; + +/// The attribute holding the Trade Desk ID `LiveIntent` resolved. +const TDID_ATTRIBUTE: &str = "thetradedesk"; + +/// `matcher` of the Trade Desk ID EID: `LiveIntent` matched it. It is also +/// the Trade Desk ID's `ext.provider` when the publisher domain is unknown, +/// as in Prebid. +const TDID_MATCHER: &str = "liveintent.com"; + +/// `LiveIntent`'s default first-party ID cookie on the publisher's domain. +const FIRST_PARTY_COOKIE: &str = "_lc2_fpi"; + +/// `atype` `LiveIntent` IDs carry: a person-based ID. +const LIVEINTENT_ATYPE: i32 = 3; + +/// The attribute holding `LiveIntent`'s own ID; the others are partner IDs. +const NON_ID_ATTRIBUTE: &str = "nonId"; + +/// UID `ext` Prebid puts on partner IDs that `LiveIntent` resolved. +const PARTNER_UID_EXT: &str = r#"{"provider":"liveintent.com"}"#; + +/// Prefix of a `LiveIntent`-issued distributor ID, such as `did-0123`. +const DISTRIBUTOR_ID_PREFIX: &str = "did-"; + +/// Most UIDs accepted per `LiveIntent` EID. +const MAX_UIDS_PER_EID: NonZeroUsize = NonZeroUsize::new(1).expect("should be non-zero"); + +/// The `LiveIntent` provider kind. +pub struct LiveIntent; + +/// Settings for a `liveintent` provider. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LiveIntentSettings { + /// The `LiveIntent` publisher ID; `LiveIntent` accepts `any`. + #[serde(default = "default_publisher_id")] + pub publisher_id: String, + /// The `LiveIntent`-issued distributor ID, such as `did-0123`, when the + /// publisher has one. + #[serde(default)] + pub distributor_id: Option, + /// The attributes to resolve, from [`ATTRIBUTE_SOURCES`]. Each one's + /// source must be one of the provider's configured `sources`. + #[serde(default = "default_resolve")] + pub resolve: Vec, + /// First-party `LiveIntent` cookies, in order; the first present one is + /// sent as `duid`. + #[serde(default = "default_cookie_names")] + pub cookie_names: Vec, +} + +fn default_publisher_id() -> String { + "any".to_owned() +} + +fn default_resolve() -> Vec { + vec![NON_ID_ATTRIBUTE.to_owned()] +} + +fn default_cookie_names() -> Vec { + vec![FIRST_PARTY_COOKIE.to_owned()] +} + +/// A built `LiveIntent` provider. +#[derive(Debug)] +pub struct LiveIntentProvider { + capabilities: ProviderCapabilities, + url: Url, + distributor_id: Option, + attributes: Box<[(String, SourceDomain)]>, + cookie_names: Box<[String]>, + partner_uid_ext: &'static RawValue, +} + +/// Whether `id` has the `did-` form `LiveIntent` accepts; it answers any +/// other distributor ID with `400 Malformed did`. +fn is_distributor_id(id: &str) -> bool { + id.strip_prefix(DISTRIBUTOR_ID_PREFIX).is_some_and(|rest| { + !rest.is_empty() && rest.bytes().all(|byte| byte.is_ascii_alphanumeric()) + }) +} + +impl ProviderKind for LiveIntent { + const KIND: &'static str = "liveintent"; + type Settings = LiveIntentSettings; + type Provider = LiveIntentProvider; + + fn build( + common: &ProviderCommon<'_>, + settings: LiveIntentSettings, + ) -> Result> { + let attributes = settings + .resolve + .into_iter() + .map(|attribute| { + let (_, source) = ATTRIBUTE_SOURCES + .iter() + .find(|(name, _)| *name == attribute) + .ok_or_else(|| { + Report::new(ProviderConfigError::Invalid) + .attach(format!("unknown LiveIntent attribute `{attribute}`")) + })?; + let source = common + .sources + .iter() + .find(|known| known.as_str() == *source) + .ok_or_else(|| { + Report::new(ProviderConfigError::Invalid).attach(format!( + "attribute `{attribute}` maps to `{source}`, which is not a configured source" + )) + })?; + Ok((attribute, source.clone())) + }) + .collect::, Report>>()?; + let max_eids = NonZeroUsize::new(attributes.len()).ok_or_else(|| { + Report::new(ProviderConfigError::Invalid) + .attach("a liveintent provider needs at least one attribute") + })?; + if let Some(id) = settings + .distributor_id + .as_deref() + .filter(|id| !is_distributor_id(id)) + { + return Err(Report::new(ProviderConfigError::Invalid).attach(format!( + "distributor_id `{id}` is not a LiveIntent `did-` distributor ID" + ))); + } + let partner_uid_ext = serde_json::from_str::<&'static RawValue>(PARTNER_UID_EXT) + .change_context(ProviderConfigError::Invalid)?; + let distributor = settings.distributor_id.as_deref().unwrap_or("prebid"); + let mut url = common + .endpoint + .ok_or_else(|| { + Report::new(ProviderConfigError::Invalid) + .attach("a liveintent provider needs an endpoint") + })? + .clone(); + url.path_segments_mut() + .map_err(|()| { + Report::new(ProviderConfigError::Invalid).attach("endpoint cannot hold a path") + })? + .pop_if_empty() + .push(distributor) + .push(&settings.publisher_id); + Ok(LiveIntentProvider { + capabilities: ProviderCapabilities { + sources: attributes + .iter() + .map(|(_, source)| SourceCapability { + source: source.clone(), + input: SourceInput::Anonymous, + }) + .collect(), + shape: EidShape { + max_eids, + max_uids_per_eid: MAX_UIDS_PER_EID, + eid_level_fields: EidLevelFields::Preserved, + }, + }, + url, + distributor_id: settings.distributor_id, + attributes, + cookie_names: settings.cookie_names.into_boxed_slice(), + partner_uid_ext, + }) + } +} + +/// Reads the `Expires` header as a bounded TTL. +/// +/// The TTL is `Expires` minus the response's `Date`, so it needs no local +/// clock. It is bounded by [`bounded_expiry`]: a missing or unparseable +/// `Expires` or `Date` gets the default TTL, so a UID never lives forever, +/// and an `Expires` at or before `Date` keeps the UID for the minimum TTL +/// instead of dropping a valid ID. +fn expiry(headers: &HeaderMap) -> ProviderExpiry { + let date = |name| -> Option { + let value = headers.get(name)?.to_str().ok()?; + httpdate::parse_http_date(value).ok() + }; + let ttl = date(EXPIRES) + .zip(date(DATE)) + .map(|(expires, date)| expires.duration_since(date).unwrap_or(Duration::ZERO)); + bounded_expiry(ttl) +} + +/// Whether a `liveintent` provider excludes `source` where GDPR applies: the +/// UID2 tokens it resolves, matching the `uid2` provider. +pub(crate) fn excludes_where_gdpr_applies(source: &str) -> bool { + source.eq_ignore_ascii_case(UID2_SOURCE) +} + +impl IdentityProvider for LiveIntentProvider { + fn capabilities(&self) -> &ProviderCapabilities { + &self.capabilities + } + + /// UID2 tokens `LiveIntent` resolves are not collected or kept where GDPR + /// applies, matching the `uid2` provider. + fn excluded_where_gdpr_applies(&self, source: &SourceDomain) -> bool { + excludes_where_gdpr_applies(source.as_str()) + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + let consent = lookup.consent; + let origin = format!("https://{}", lookup.publisher_domain); + let mut url = self.url.clone(); + { + let mut query = url.query_pairs_mut(); + if let Some(distributor) = &self.distributor_id { + query.append_pair("did", distributor); + } + if let Some(first_party_id) = self + .cookie_names + .iter() + .find_map(|name| lookup.cookies.get(name)) + { + query.append_pair("duid", first_party_id); + } + query.append_pair("gdpr", if consent.gdpr_applies { "1" } else { "0" }); + if let Some(tc_string) = &consent.raw_tc_string { + query.append_pair("gdpr_consent", tc_string); + } + if let Some(us_privacy) = &consent.raw_us_privacy { + query.append_pair("us_privacy", us_privacy); + } + if let Some(gpp) = &consent.raw_gpp_string { + query.append_pair("gpp_s", gpp); + } + if let Some(ids) = &consent.gpp_section_ids { + query.append_pair("gpp_as", &join_section_ids(ids)); + } + query.append_pair("pu", &origin); + if let Some(ip) = lookup.client.ip.filter(|ip| is_routable_ip(*ip)) { + let name = if ip.is_ipv4() { "pip" } else { "pip6" }; + query.append_pair(name, &URL_SAFE_NO_PAD.encode(ip.to_string())); + } + for (attribute, source) in &self.attributes { + if lookup.due_sources.contains(source) { + query.append_pair("resolve", attribute); + } + } + } + let mut headers = HeaderMap::new(); + headers.insert( + ORIGIN, + HeaderValue::from_str(&origin).change_context(ProviderError::Request)?, + ); + Ok(ProviderRequest { + method: Method::GET, + url, + headers, + body: Bytes::new(), + // The publisher domain names the Trade Desk ID's provider. + context: Some(lookup.publisher_domain.to_owned()), + }) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + if response.status == StatusCode::NO_CONTENT { + return Ok(LookupOutcome::NoMatch); + } + if !response.status.is_success() { + return Err(Report::new(ProviderError::InvalidResponse) + .attach(format!("LiveIntent returned status {}", response.status))); + } + let reply: HashMap = + serde_json::from_slice(response.body).change_context(ProviderError::InvalidResponse)?; + let expiry = expiry(response.headers); + let non_id_ext = non_id_ext(&reply); + let tdid_uid_ext = tdid_uid_ext(response.context)?; + let eids: Vec> = self + .attributes + .iter() + .filter_map(|(attribute, source)| { + let raw = reply.get(attribute.as_str())?; + let id = Cow::<'body, str>::deserialize(*raw) + .ok() + .filter(|id| is_usable_id(id))?; + // Prebid sends the Trade Desk ID LiveIntent resolved as a + // device-based ID (atype 1), like Unified ID's TDID. + let (matcher, eid_ext, uid_ext, atype) = match attribute.as_str() { + NON_ID_ATTRIBUTE => ( + None, + non_id_ext.clone().map(Cow::Owned), + None, + LIVEINTENT_ATYPE, + ), + TDID_ATTRIBUTE => ( + Some(Cow::Borrowed(TDID_MATCHER)), + None, + Some(Cow::Owned(tdid_uid_ext.clone())), + DEVICE_ATYPE, + ), + _ => ( + None, + None, + Some(Cow::Borrowed(self.partner_uid_ext)), + LIVEINTENT_ATYPE, + ), + }; + Some(ProviderEid { + source: Cow::Owned(source.as_str().to_owned()), + inserter: None, + matcher, + mm: None, + ext: eid_ext, + uids: vec![ProviderUid { + id, + atype: Some(atype), + ext: uid_ext, + expiry, + refresh: None, + }], + }) + }) + .collect(); + Ok(if eids.is_empty() { + LookupOutcome::NoMatch + } else { + LookupOutcome::Found(eids) + }) + } +} + +/// Builds the `nonId` EID `ext` from the reply's `segments` and `m`, as +/// Prebid's `LiveIntent` module does. +/// +/// Only a non-empty `segments` array and a truthy `m` are kept. An +/// oversized `ext` would drop the whole `nonId` EID, so the segments are +/// left out first, then `m`, rather than the ID. +fn non_id_ext(reply: &HashMap) -> Option> { + let field = |name: &str| -> Option { + serde_json::from_str(reply.get(name)?.get()).ok() + }; + let segments = field("segments").filter(|segments| { + segments + .as_array() + .is_some_and(|segments| !segments.is_empty()) + }); + let m = field("m").filter(|m| { + !matches!(m, serde_json::Value::Null | serde_json::Value::Bool(false)) + && m.as_str() != Some("") + && m.as_f64() != Some(0.0) + }); + let build = |segments: Option<&serde_json::Value>| -> Option> { + let mut ext = serde_json::Map::new(); + if let Some(segments) = segments { + ext.insert("segments".to_owned(), segments.clone()); + } + if let Some(m) = &m { + ext.insert("m".to_owned(), m.clone()); + } + if ext.is_empty() { + return None; + } + let ext = serde_json::value::to_raw_value(&ext).ok()?; + if ext.get().len() > MAX_EXT_BYTES { + log::debug!("LiveIntent: omitting a nonId EID ext over {MAX_EXT_BYTES} bytes"); + return None; + } + Some(ext) + }; + build(segments.as_ref()).or_else(|| segments.as_ref().and_then(|_| build(None))) +} + +/// Builds the Trade Desk ID's UID `ext`: Prebid names the page's domain as +/// its `provider`, falling back to `LiveIntent`'s domain. +/// +/// # Errors +/// +/// Returns [`ProviderError::InvalidResponse`] if the `ext` cannot be +/// serialized. +fn tdid_uid_ext(publisher_domain: Option<&str>) -> Result, Report> { + let provider = publisher_domain + .filter(|domain| !domain.is_empty()) + .unwrap_or(TDID_MATCHER); + serde_json::value::to_raw_value(&serde_json::json!({ + "rtiPartner": "TDID", + "provider": provider, + })) + .change_context(ProviderError::InvalidResponse) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + use trusted_server_core::consent::types::ConsentContext; + use trusted_server_core::ec::provider::{ + ClientSignals, OwnedEntries, RequestCookies, build_kind, + }; + use trusted_server_core::settings::{IdentityProviderConfig, ProviderSecrets}; + + use super::*; + + const SOURCES: [&str; 3] = ["liveintent.com", "adserver.org", "pubmatic.com"]; + + fn config(resolve: &serde_json::Value) -> IdentityProviderConfig { + serde_json::from_value(json!({ + "id": "liveintent", + "kind": "liveintent", + "endpoint": "https://li.example.com/idex", + "allowed_domains": ["li.example.com"], + "sources": SOURCES, + "settings": { "publisher_id": "12345", "resolve": resolve }, + })) + .expect("should parse provider config") + } + + fn provider() -> Box { + build_kind::( + &config(&json!(["nonId", "thetradedesk", "pubmatic"])), + &ProviderSecrets::default(), + ) + .expect("should build provider") + } + + fn due() -> Vec { + SOURCES + .into_iter() + .map(|source| SourceDomain::try_from(source).expect("should parse source")) + .collect() + } + + #[test] + fn rejects_an_unknown_attribute_or_one_whose_source_is_not_configured() { + for resolve in [json!(["unknown"]), json!(["bidswitch"])] { + let error = build_kind::(&config(&resolve), &ProviderSecrets::default()) + .err() + .expect("should reject the config"); + + assert!( + matches!(error.current_context(), ProviderConfigError::Invalid), + "should report invalid config for {resolve}" + ); + } + } + + #[test] + fn excludes_only_uid2_where_gdpr_applies() { + let provider = provider(); + let uid2 = SourceDomain::try_from("uidapi.com").expect("should build source"); + let nonid = SourceDomain::try_from("liveintent.com").expect("should build source"); + + assert!( + provider.excluded_where_gdpr_applies(&uid2), + "should exclude UID2 tokens where GDPR applies" + ); + assert!( + !provider.excluded_where_gdpr_applies(&nonid), + "should keep LiveIntent's own ID where GDPR applies" + ); + } + + #[test] + fn sends_the_first_present_first_party_cookie_as_duid() { + let provider = provider(); + let consent = ConsentContext::default(); + let due = due(); + let lookup = |cookies| LookupInput { + ec_id: "ec-example", + consent: &consent, + client: ClientSignals { + ip: None, + user_agent: None, + }, + publisher_domain: "www.example.com", + due_sources: &due, + current: OwnedEntries::empty(), + cookies, + hashed_email: None, + now: trusted_server_core::ec::eid_types::UnixSeconds::new(1_700_000_000), + }; + + for (header, expected) in [ + ("other=1; _lc2_fpi=fpi-example", vec!["fpi-example"]), + ("other=1; _lc2_fpi=", vec![]), + ("", vec![]), + ] { + let request = provider + .build_request(&lookup(RequestCookies::new(header))) + .expect("should build request"); + let duid: Vec = request + .url + .query_pairs() + .filter(|(key, _)| key == "duid") + .map(|(_, value)| value.into_owned()) + .collect(); + + assert_eq!(duid, expected, "should send duid for `{header}`"); + } + } + + #[test] + fn calls_the_exchange_with_consent_ip_and_due_attributes() { + let provider = provider(); + let consent = ConsentContext { + raw_us_privacy: Some("1YNN".to_owned()), + ..ConsentContext::default() + }; + let due = due(); + let lookup = LookupInput { + ec_id: "ec-example", + consent: &consent, + client: ClientSignals { + ip: Some("11.22.33.44".parse().expect("should parse ip")), + user_agent: None, + }, + publisher_domain: "www.example.com", + due_sources: &due[..1], + current: OwnedEntries::empty(), + cookies: RequestCookies::default(), + hashed_email: None, + now: trusted_server_core::ec::eid_types::UnixSeconds::new(1_700_000_000), + }; + + let request = provider + .build_request(&lookup) + .expect("should build request"); + let query: Vec<(String, String)> = request + .url + .query_pairs() + .map(|(key, value)| (key.into_owned(), value.into_owned())) + .collect(); + let value = |name: &str| { + query + .iter() + .filter(|(key, _)| key == name) + .map(|(_, value)| value.as_str()) + .collect::>() + }; + + assert_eq!(request.method, Method::GET, "should use GET"); + assert_eq!( + request.url.path(), + "/idex/prebid/12345", + "should call the distributor and publisher path" + ); + assert_eq!(value("gdpr"), ["0"], "should send that GDPR does not apply"); + assert_eq!(value("us_privacy"), ["1YNN"], "should send US privacy"); + assert_eq!( + value("pu"), + ["https://www.example.com"], + "should send the origin" + ); + assert_eq!( + value("pip"), + [URL_SAFE_NO_PAD.encode("11.22.33.44").as_str()], + "should send the encoded IP" + ); + assert_eq!( + value("resolve"), + ["nonId"], + "should ask only for due attributes" + ); + } + + #[test] + fn omits_empty_or_null_segments() { + let provider = provider(); + let headers = HeaderMap::new(); + + for segments in [json!([]), json!(null), json!("s1")] { + let body = json!({ "nonId": "li-example", "segments": segments }) + .to_string() + .into_bytes(); + let outcome = provider + .parse_response(ProviderResponse { + status: StatusCode::OK, + headers: &headers, + body: &body, + context: None, + }) + .expect("should parse response"); + let LookupOutcome::Found(eids) = outcome else { + panic!("should find the nonId EID"); + }; + + assert!( + eids[0].ext.is_none(), + "should omit segments {segments} from the nonId EID ext" + ); + } + } + + #[test] + fn omits_oversized_segments_but_keeps_the_non_id() { + let provider = provider(); + let headers = HeaderMap::new(); + let segments: Vec = (0..120).map(|n| format!("seg{n:04}")).collect(); + let body = json!({ "nonId": "li-example", "segments": segments }) + .to_string() + .into_bytes(); + + let outcome = provider + .parse_response(ProviderResponse { + status: StatusCode::OK, + headers: &headers, + body: &body, + context: None, + }) + .expect("should parse response"); + let LookupOutcome::Found(eids) = outcome else { + panic!("should find the nonId EID"); + }; + + assert_eq!(eids[0].uids[0].id, "li-example", "should keep the ID"); + assert!( + eids[0].ext.is_none(), + "should omit segments that exceed the ext limit" + ); + } + + #[test] + fn maps_returned_attributes_to_sources_with_the_expires_header() { + let provider = provider(); + let mut headers = HeaderMap::new(); + headers.insert( + EXPIRES, + HeaderValue::from_static("Sun, 06 Nov 1994 10:49:37 GMT"), + ); + headers.insert( + DATE, + HeaderValue::from_static("Sun, 06 Nov 1994 08:49:37 GMT"), + ); + // Escape the `nonId` key as `LiveIntent` may, which a borrowed key + // cannot hold. + let body = json!({ + "nonId": "li-example", + "segments": ["s1", "s2"], + "thetradedesk": "tdid-example", + "pubmatic": "pubmatic-example", + "unknown": "x", + }) + .to_string() + .replace("\"nonId\"", "\"non\\u0049d\"") + .into_bytes(); + + let outcome = provider + .parse_response(ProviderResponse { + status: StatusCode::OK, + headers: &headers, + body: &body, + context: None, + }) + .expect("should parse response"); + let LookupOutcome::Found(eids) = outcome else { + panic!("should find EIDs"); + }; + + assert_eq!(eids.len(), 3, "should return each configured attribute"); + assert_eq!(eids[0].source, "liveintent.com", "should map nonId"); + assert_eq!(eids[0].uids[0].id, "li-example", "should keep the ID"); + assert_eq!(eids[0].uids[0].atype, Some(3), "should set atype 3"); + assert!( + matches!(eids[0].uids[0].expiry, ProviderExpiry::After(d) if d == Duration::from_secs(7200)), + "should use Expires relative to Date" + ); + assert!( + eids[0].uids[0].ext.is_none(), + "should keep nonId without ext" + ); + assert_eq!( + eids[0].ext.as_deref().map(RawValue::get), + Some(r#"{"segments":["s1","s2"]}"#), + "should keep segments as the nonId EID ext" + ); + assert!( + eids[0].matcher.is_none(), + "should keep nonId without matcher" + ); + assert_eq!(eids[1].source, "adserver.org", "should map the TDID"); + assert_eq!( + eids[1].uids[0].atype, + Some(1), + "should send the TDID as device-based, as Prebid does" + ); + assert_eq!( + eids[1].matcher.as_deref(), + Some("liveintent.com"), + "should name LiveIntent as the TDID matcher" + ); + assert_eq!( + eids[1].uids[0] + .ext + .as_deref() + .map(|ext| serde_json::from_str::(ext.get()) + .expect("should parse TDID ext")), + Some(json!({ "rtiPartner": "TDID", "provider": "liveintent.com" })), + "should fall back to LiveIntent as the TDID provider without a publisher domain" + ); + assert!(eids[1].ext.is_none(), "should keep segments on nonId only"); + assert_eq!(eids[2].source, "pubmatic.com", "should map a partner ID"); + assert_eq!( + eids[2].uids[0].ext.as_deref().map(RawValue::get), + Some(r#"{"provider":"liveintent.com"}"#), + "should name LiveIntent as the provider of a partner ID" + ); + } + + #[test] + fn names_the_publisher_domain_as_the_tdid_provider() { + let provider = provider(); + let headers = HeaderMap::new(); + let consent = ConsentContext::default(); + let due = due(); + let lookup = LookupInput { + ec_id: "ec-example", + consent: &consent, + client: ClientSignals { + ip: None, + user_agent: None, + }, + publisher_domain: "www.example.com", + due_sources: &due, + current: OwnedEntries::empty(), + cookies: RequestCookies::default(), + hashed_email: None, + now: trusted_server_core::ec::eid_types::UnixSeconds::new(1_700_000_000), + }; + let request = provider + .build_request(&lookup) + .expect("should build request"); + let body = json!({ "thetradedesk": "tdid-example" }) + .to_string() + .into_bytes(); + + let outcome = provider + .parse_response(ProviderResponse { + status: StatusCode::OK, + headers: &headers, + body: &body, + context: request.context.as_deref(), + }) + .expect("should parse response"); + let LookupOutcome::Found(eids) = outcome else { + panic!("should find the TDID EID"); + }; + + let ext: serde_json::Value = serde_json::from_str( + eids[0].uids[0] + .ext + .as_deref() + .expect("should set the TDID ext") + .get(), + ) + .expect("should parse TDID ext"); + assert_eq!( + ext, + json!({ "rtiPartner": "TDID", "provider": "www.example.com" }), + "should name the publisher domain as the TDID provider, as Prebid does" + ); + assert_eq!( + eids[0].matcher.as_deref(), + Some("liveintent.com"), + "should keep LiveIntent as the TDID matcher" + ); + } + + #[test] + fn copies_m_into_the_non_id_ext() { + let provider = provider(); + let headers = HeaderMap::new(); + let cases = [ + ( + json!({ "nonId": "li-example", "segments": ["s1"], "m": "m-example" }), + Some(json!({ "segments": ["s1"], "m": "m-example" })), + ), + ( + json!({ "nonId": "li-example", "m": "m-example" }), + Some(json!({ "m": "m-example" })), + ), + (json!({ "nonId": "li-example", "m": "" }), None), + (json!({ "nonId": "li-example", "m": null }), None), + ( + json!({ + "nonId": "li-example", + "segments": (0..120).map(|n| format!("seg{n:04}")).collect::>(), + "m": "m-example", + }), + Some(json!({ "m": "m-example" })), + ), + ]; + + for (reply, expected) in cases { + let body = reply.to_string().into_bytes(); + let outcome = provider + .parse_response(ProviderResponse { + status: StatusCode::OK, + headers: &headers, + body: &body, + context: None, + }) + .expect("should parse response"); + let LookupOutcome::Found(eids) = outcome else { + panic!("should find the nonId EID"); + }; + + let ext = eids[0].ext.as_deref().map(|ext| { + serde_json::from_str::(ext.get()) + .expect("should parse nonId ext") + }); + assert_eq!(ext, expected, "should build the nonId ext for {reply}"); + } + } + + #[test] + fn rejects_a_distributor_id_liveintent_did_not_issue() { + for distributor_id in ["prebid", "did-", "did-01/2"] { + let mut config = config(&json!(["nonId"])); + config + .settings + .insert("distributor_id".to_owned(), json!(distributor_id)); + + let error = build_kind::(&config, &ProviderSecrets::default()) + .err() + .expect("should reject the distributor ID"); + + assert!( + matches!(error.current_context(), ProviderConfigError::Invalid), + "should report invalid config for `{distributor_id}`" + ); + } + } + + #[test] + fn calls_the_distributor_path_with_a_did_distributor_id() { + let mut config = config(&json!(["nonId"])); + config + .settings + .insert("distributor_id".to_owned(), json!("did-0123")); + let provider = build_kind::(&config, &ProviderSecrets::default()) + .expect("should build provider"); + let consent = ConsentContext::default(); + let due = due(); + let lookup = LookupInput { + ec_id: "ec-example", + consent: &consent, + client: ClientSignals { + ip: None, + user_agent: None, + }, + publisher_domain: "www.example.com", + due_sources: &due, + current: OwnedEntries::empty(), + cookies: RequestCookies::default(), + hashed_email: None, + now: trusted_server_core::ec::eid_types::UnixSeconds::new(1_700_000_000), + }; + + let request = provider + .build_request(&lookup) + .expect("should build request"); + + assert_eq!( + request.url.path(), + "/idex/did-0123/12345", + "should call the distributor path" + ); + assert!( + request + .url + .query_pairs() + .any(|(key, value)| key == "did" && value == "did-0123"), + "should send the distributor ID" + ); + } + + #[test] + fn bounds_a_missing_or_past_expires() { + let mut past = HeaderMap::new(); + past.insert( + EXPIRES, + HeaderValue::from_static("Sun, 06 Nov 1994 08:49:37 GMT"), + ); + past.insert( + DATE, + HeaderValue::from_static("Mon, 07 Nov 1994 08:49:37 GMT"), + ); + + for (headers, expected) in [ + (HeaderMap::new(), crate::common::DEFAULT_TTL), + (past, crate::common::MIN_TTL), + ] { + assert!( + matches!(expiry(&headers), ProviderExpiry::After(d) if d == expected), + "should bound the expiry to {expected:?}" + ); + } + } + + #[test] + fn reports_no_match_for_no_content_and_empty_ids() { + let provider = provider(); + let headers = HeaderMap::new(); + + let empty_id = serde_json::to_vec(&json!({ "nonId": "" })).expect("should encode body"); + let blank_id = serde_json::to_vec(&json!({ "nonId": " " })).expect("should encode body"); + let zero_id = serde_json::to_vec(&json!({ "nonId": "0" })).expect("should encode body"); + for (status, body) in [ + (StatusCode::NO_CONTENT, &b""[..]), + (StatusCode::OK, &empty_id[..]), + (StatusCode::OK, &blank_id[..]), + (StatusCode::OK, &zero_id[..]), + ] { + let outcome = provider + .parse_response(ProviderResponse { + status, + headers: &headers, + body, + context: None, + }) + .expect("should parse response"); + assert!( + matches!(outcome, LookupOutcome::NoMatch), + "should report no match" + ); + } + } + + #[test] + fn rejects_an_error_status_and_a_malformed_body() { + let provider = provider(); + let headers = HeaderMap::new(); + + for (status, body) in [ + (StatusCode::BAD_GATEWAY, &b"{}"[..]), + (StatusCode::OK, b"not json"), + ] { + let error = provider + .parse_response(ProviderResponse { + status, + headers: &headers, + body, + context: None, + }) + .expect_err("should reject the response"); + assert!( + matches!(error.current_context(), ProviderError::InvalidResponse), + "should report an invalid response" + ); + } + } +} diff --git a/crates/trusted-server-identity-providers/src/liveramp.rs b/crates/trusted-server-identity-providers/src/liveramp.rs new file mode 100644 index 000000000..9b38e4653 --- /dev/null +++ b/crates/trusted-server-identity-providers/src/liveramp.rs @@ -0,0 +1,822 @@ +//! `LiveRamp` identity provider: reads the `RampID` envelope from `LiveRamp`'s own +//! first-party cookie. +//! +//! `LiveRamp`'s ATS script stores the envelope in the `_lr_env` cookie as +//! base64 (or base64url) of a JSON object whose `envelope` field holds it. +//! Prebid's `identityLink` ID module stores the envelope it receives bare, +//! under its configured storage name (commonly `idl_env`); a value that does +//! not decode to JSON is taken as such a bare envelope. +//! The envelope is stored unchanged with agent type 3 for +//! [`ENVELOPE_TTL`]. +//! +//! With an `endpoint` (`LiveRamp`'s ATS envelope API, path +//! `/api/identity/v2/envelope`), a visitor without an envelope cookie is looked up server side by Custom ID (`it=15`): the +//! stable part of the EC ID, sent as `iv=:` with +//! the placement ID, consent strings, the publisher `Origin` and the +//! visitor's IP in `X-Forwarded-For`, as a server-to-server call +//! (`atype=3`). `LiveRamp` must enable Custom ID for the placement, and it +//! resolves a Custom ID only through a mapping file the publisher sends it, +//! tying each Custom ID to a hashed email; without one the lookup finds +//! nothing. A `200` +//! reply lists `envelopes`; the `RampID` entry (`type` 19) is stored. A +//! `204` reply reports an opt-out. Without an `endpoint` no server call is +//! made. +//! +//! With the Lockr capability enabled (`[settings.lockr]`), the cookies Lockr +//! AIM writes the envelope into (default key `_lr_env`) are read after the +//! own cookies, with the same parser. +//! +//! Parsing is lenient: a value outside the base64 alphabet, a JSON body +//! without a string `envelope`, or a blank or placeholder (`0`) envelope reads +//! as no ID. + +use std::borrow::Cow; +use std::time::Duration; + +use bytes::Bytes; +use http::header::ORIGIN; +use http::{HeaderMap, HeaderName, HeaderValue, Method, StatusCode}; +use url::Url; + +use base64::Engine as _; +use base64::engine::general_purpose::{STANDARD, STANDARD_NO_PAD, URL_SAFE, URL_SAFE_NO_PAD}; +use error_stack::{Report, ResultExt as _}; +use serde::Deserialize; +use trusted_server_core::ec::eid_types::SourceDomain; +use trusted_server_core::ec::provider::{ + CookieRead, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, + ProviderConfigError, ProviderError, ProviderKind, ProviderRequest, ProviderResponse, + RequestCookies, +}; + +use crate::common::{ + CookieReader, LockrSettings, ReadsLockr, bounded_expiry, cookie_capabilities, cookie_eid, + is_routable_ip, is_usable_id, percent_decoded, +}; + +/// `it` value of a Custom ID lookup. +const CUSTOM_ID_TYPE: &str = "15"; +/// `atype` value of a server-to-server integration (`2` is the ATS API). +const SERVER_TO_SERVER: &str = "3"; +/// `type` of the `RampID` envelope in a v2 reply's `envelopes`. +const RAMPID_ENVELOPE_TYPE: u32 = 19; +/// `ct` value of a TCF v2 consent string. +const TCF_CONSENT: &str = "4"; +/// `ct` value of a US Privacy consent string. +const US_PRIVACY_CONSENT: &str = "3"; + +/// How long a stored envelope is kept. +pub const ENVELOPE_TTL: Duration = Duration::from_secs(15 * 24 * 60 * 60); + +/// The `LiveRamp` provider kind. +pub struct LiveRamp; + +/// Settings for a `liveramp` provider. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LiveRampSettings { + /// Cookies read in order; the first holding an envelope wins. + #[serde(default = "LiveRampSettings::default_cookie_names")] + pub cookie_names: Vec, + /// The Lockr capability; see [`LockrSettings`]. + #[serde(default)] + pub lockr: LockrSettings, + /// `LiveRamp` Placement ID (`pid`); required with an `endpoint`. + #[serde(default)] + pub placement_id: Option, + /// `LiveRamp`-assigned namespace of the Custom ID (the `ANA_ID` part of + /// `iv`); required with an `endpoint`. + #[serde(default)] + pub custom_id_namespace: Option, +} + +impl LiveRampSettings { + fn default_cookie_names() -> Vec { + vec!["_lr_env".to_owned(), "idl_env".to_owned()] + } +} + +/// A built `LiveRamp` provider. +#[derive(Debug)] +pub struct LiveRampProvider { + capabilities: ProviderCapabilities, + source: SourceDomain, + cookies: CookieReader, + /// The envelope lookup; `None` disables server calls. + lookup: Option, +} + +/// A configured Custom ID envelope lookup. +#[derive(Debug)] +struct EnvelopeLookup { + endpoint: Url, + placement_id: u64, + custom_id_namespace: String, +} + +impl ReadsLockr for LiveRamp { + const LOCKR_COOKIE_NAMES: &'static [&'static str] = &["_lr_env"]; +} + +impl ProviderKind for LiveRamp { + const KIND: &'static str = "liveramp"; + type Settings = LiveRampSettings; + type Provider = LiveRampProvider; + + fn build( + common: &ProviderCommon<'_>, + settings: LiveRampSettings, + ) -> Result> { + let cookies = + CookieReader::with_lockr::(common, settings.cookie_names, settings.lockr)?; + if cookies.is_empty() { + return Err(Report::new(ProviderConfigError::Invalid) + .attach("a liveramp provider needs at least one cookie name")); + } + let (capabilities, source) = cookie_capabilities(common, Self::KIND)?; + let lookup = match ( + common.endpoint, + settings.placement_id, + settings.custom_id_namespace, + ) { + (None, _, _) => None, + (Some(endpoint), Some(placement_id), Some(custom_id_namespace)) + if !custom_id_namespace.trim().is_empty() => + { + Some(EnvelopeLookup { + endpoint: endpoint.clone(), + placement_id, + custom_id_namespace, + }) + } + (Some(_), _, _) => { + return Err(Report::new(ProviderConfigError::Invalid) + .attach("a liveramp endpoint needs placement_id and custom_id_namespace")); + } + }; + Ok(LiveRampProvider { + capabilities, + source, + cookies, + lookup, + }) + } +} + +#[derive(Deserialize)] +struct EnvelopeCookie { + envelope: String, +} + +/// A v2 envelope API reply: `{"envelopes":[{"type":19,"value":…}]}`. +#[derive(Deserialize)] +struct EnvelopeReply { + #[serde(default)] + envelopes: Vec, +} + +/// One entry of [`EnvelopeReply::envelopes`]. +#[derive(Deserialize)] +struct ReplyEnvelope { + #[serde(rename = "type", default)] + kind: Option, + #[serde(default)] + value: Option, +} + +/// Decodes one cookie value into its envelope. +/// +/// A value whose base64 decodes to JSON must carry a usable `envelope` +/// (the ATS `_lr_env` format); any other base64-alphabet value is a bare +/// envelope (the Prebid `idl_env` format). +fn envelope(value: &str) -> Option { + let value = percent_decoded(value); + let value = value.trim(); + let is_base64_alphabet = value.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'+' | b'/' | b'=' | b'-' | b'_') + }); + if !is_base64_alphabet { + return None; + } + let json = [STANDARD, STANDARD_NO_PAD, URL_SAFE, URL_SAFE_NO_PAD] + .iter() + .filter_map(|engine| engine.decode(value).ok()) + .find_map(|bytes| serde_json::from_slice::(&bytes).ok()); + let envelope = match json { + Some(body) => { + serde_json::from_value::(body) + .ok()? + .envelope + } + None => value.to_owned(), + }; + is_usable_id(&envelope).then_some(envelope) +} + +impl IdentityProvider for LiveRampProvider { + fn capabilities(&self) -> &ProviderCapabilities { + &self.capabilities + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + let Some(envelope_lookup) = &self.lookup else { + return Err(Report::new(ProviderError::Request) + .attach("liveramp looks up only with an endpoint")); + }; + // LiveRamp requires the visitor's IP on server-to-server calls. + let ip = lookup + .client + .ip + .filter(|ip| is_routable_ip(*ip)) + .ok_or_else(|| { + Report::new(ProviderError::Request).attach("liveramp needs a routable client IP") + })?; + let stable_id = lookup.ec_id.split('.').next().unwrap_or(lookup.ec_id); + let consent = lookup.consent; + let mut url = envelope_lookup.endpoint.clone(); + { + let mut query = url.query_pairs_mut(); + query + .append_pair("pid", &envelope_lookup.placement_id.to_string()) + .append_pair("it", CUSTOM_ID_TYPE) + .append_pair( + "iv", + &format!("{}:{stable_id}", envelope_lookup.custom_id_namespace), + ) + .append_pair("atype", SERVER_TO_SERVER); + if let Some(tc_string) = &consent.raw_tc_string { + query + .append_pair("ct", TCF_CONSENT) + .append_pair("cv", tc_string); + } else if let Some(us_privacy) = &consent.raw_us_privacy { + query + .append_pair("ct", US_PRIVACY_CONSENT) + .append_pair("cv", us_privacy); + } + if let Some(gpp) = &consent.raw_gpp_string { + query.append_pair("gpp", gpp); + } + // `gpp_sid` is a single section ID; Prebid also sends only the + // first applicable section. + if let Some(section_id) = consent.gpp_section_ids.as_deref().and_then(<[u16]>::first) { + query.append_pair("gpp_sid", §ion_id.to_string()); + } + } + let mut headers = HeaderMap::new(); + headers.insert( + ORIGIN, + HeaderValue::from_str(&format!("https://{}", lookup.publisher_domain)) + .change_context(ProviderError::Request)?, + ); + headers.insert( + HeaderName::from_static("x-forwarded-for"), + HeaderValue::from_str(&ip.to_string()).change_context(ProviderError::Request)?, + ); + Ok(ProviderRequest { + method: Method::GET, + url, + headers, + body: Bytes::new(), + context: None, + }) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + if response.status == StatusCode::NO_CONTENT { + return Ok(LookupOutcome::OptedOut); + } + if !response.status.is_success() { + return Err(Report::new(ProviderError::InvalidResponse) + .attach(format!("liveramp returned status {}", response.status))); + } + let reply: EnvelopeReply = + serde_json::from_slice(response.body).change_context(ProviderError::InvalidResponse)?; + let Some(envelope) = reply + .envelopes + .into_iter() + .find(|envelope| envelope.kind == Some(RAMPID_ENVELOPE_TYPE)) + .and_then(|envelope| envelope.value) + .filter(|envelope| is_usable_id(envelope)) + else { + return Ok(LookupOutcome::NoMatch); + }; + Ok(LookupOutcome::Found(vec![cookie_eid( + &self.source, + Cow::Owned(envelope), + bounded_expiry(Some(ENVELOPE_TTL)), + )])) + } + + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> CookieRead<'cookie> { + let Some(envelope) = self.cookies.read(cookies, envelope) else { + return CookieRead::Absent; + }; + CookieRead::Found(vec![cookie_eid( + &self.source, + Cow::Owned(envelope), + bounded_expiry(Some(ENVELOPE_TTL)), + )]) + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + use trusted_server_core::consent::types::ConsentContext; + use trusted_server_core::ec::eid_types::{EidValidationError, UidValue}; + use trusted_server_core::ec::kv_types::MAX_STORED_UID_LENGTH; + use trusted_server_core::ec::provider::{ + ClientSignals, OwnedEntries, ProviderExpiry, build_kind, + }; + use trusted_server_core::settings::{IdentityProviderConfig, ProviderSecrets}; + + use super::*; + + fn provider(settings: &serde_json::Value) -> Box { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "liveramp", + "kind": "liveramp", + "sources": ["liveramp.example.com"], + "settings": settings, + })) + .expect("should parse provider config"); + build_kind::(&config, &ProviderSecrets::default()).expect("should build provider") + } + + fn encoded(body: &serde_json::Value, engine: &base64::engine::GeneralPurpose) -> String { + engine.encode(body.to_string()) + } + + fn read(provider: &dyn IdentityProvider, header: &str) -> Option<(String, ProviderExpiry)> { + let eids = provider + .read_request_cookies(RequestCookies::new(header)) + .into_found()?; + assert_eq!(eids.len(), 1, "should return one EID"); + assert_eq!( + eids[0].source, "liveramp.example.com", + "should use the source" + ); + let uid = &eids[0].uids[0]; + assert_eq!(uid.atype, Some(3), "should be person-based"); + Some((uid.id.to_string(), uid.expiry)) + } + + #[test] + fn reads_the_envelope_in_base64_and_base64url() { + let provider = provider(&json!({})); + let body = json!({ "envelope": "AjfowMv4ZHZQJFM8TpiUnYEyA81Vdgg", "ts": 1 }); + + for engine in [STANDARD, URL_SAFE_NO_PAD] { + let (id, expiry) = read(&*provider, &format!("_lr_env={}", encoded(&body, &engine))) + .expect("should read the envelope"); + + assert_eq!( + id, "AjfowMv4ZHZQJFM8TpiUnYEyA81Vdgg", + "should keep the envelope" + ); + assert!( + matches!(expiry, ProviderExpiry::After(after) if after == ENVELOPE_TTL), + "should keep for 15 days" + ); + } + } + + #[test] + fn collects_envelopes_up_to_the_uid_length_limit_and_drops_longer_ones() { + let provider = provider(&json!({})); + + for len in [703, 1000, MAX_STORED_UID_LENGTH] { + let raw = "A".repeat(len); + let ats = encoded(&json!({ "envelope": raw }), &STANDARD); + for header in [format!("idl_env={raw}"), format!("_lr_env={ats}")] { + let (id, _) = read(&*provider, &header).expect("should read the envelope"); + + assert_eq!(id.len(), len, "should keep the whole envelope"); + assert!( + UidValue::try_from(id).is_ok(), + "should pass validation at {len} bytes" + ); + } + } + + let (id, _) = read( + &*provider, + &format!("idl_env={}", "A".repeat(MAX_STORED_UID_LENGTH + 1)), + ) + .expect("should read the oversized envelope"); + assert!( + matches!(UidValue::try_from(id), Err(EidValidationError::UidTooLong)), + "should drop an envelope over the UID length limit in validation" + ); + } + + #[test] + fn reads_a_bare_prebid_idl_env_envelope() { + let provider = provider(&json!({})); + + let (id, _) = read(&*provider, "idl_env=AjfowMv4ZHZQJFM8TpiUnYEyA81Vdgg") + .expect("should read the bare envelope"); + + assert_eq!( + id, "AjfowMv4ZHZQJFM8TpiUnYEyA81Vdgg", + "should keep the bare envelope" + ); + } + + #[test] + fn prefers_lr_env_over_idl_env() { + let provider = provider(&json!({})); + let value = encoded(&json!({ "envelope": "env-ats" }), &STANDARD); + + let read = read( + &*provider, + &format!("idl_env=envPrebidExample; _lr_env={value}"), + ); + + assert_eq!( + read.map(|(id, _)| id).as_deref(), + Some("env-ats"), + "should prefer the ATS cookie" + ); + } + + #[test] + fn reads_a_configured_cookie_name() { + let provider = provider(&json!({ "cookie_names": ["example_lockr_env"] })); + let value = encoded(&json!({ "envelope": "env-example" }), &STANDARD); + + let read = read( + &*provider, + &format!("_lr_env=ignored; example_lockr_env={value}"), + ); + + assert_eq!( + read.map(|(id, _)| id).as_deref(), + Some("env-example"), + "should read it" + ); + } + + #[test] + fn ignores_lockr_cookies_while_lockr_is_disabled() { + let provider = provider(&json!({ + "lockr": { "cookie_names": ["example_lockr_env"] }, + })); + let value = encoded(&json!({ "envelope": "env-example" }), &STANDARD); + + let read = read(&*provider, &format!("example_lockr_env={value}")); + + assert!( + read.is_none(), + "should not read a Lockr cookie when disabled" + ); + } + + #[test] + fn reads_a_custom_lockr_cookie_after_the_own_cookie() { + let provider = provider(&json!({ + "lockr": { "enabled": true, "cookie_names": ["example_lockr_env"] }, + })); + let own = encoded(&json!({ "envelope": "env-own" }), &STANDARD); + let lockr = encoded(&json!({ "envelope": "env-lockr" }), &STANDARD); + + let both = read( + &*provider, + &format!("example_lockr_env={lockr}; _lr_env={own}"), + ); + let lockr_only = read(&*provider, &format!("example_lockr_env={lockr}")); + + assert_eq!( + both.map(|(id, _)| id).as_deref(), + Some("env-own"), + "should prefer the own cookie" + ); + assert_eq!( + lockr_only.map(|(id, _)| id).as_deref(), + Some("env-lockr"), + "should fall back to the Lockr cookie" + ); + } + + #[test] + fn ignores_missing_malformed_and_opt_out_cookies() { + let provider = provider(&json!({})); + for header in [ + String::new(), + "_lr_env=not-base64!".to_owned(), + "idl_env=0".to_owned(), + "idl_env=".to_owned(), + format!("_lr_env={}", encoded(&json!({ "other": "x" }), &STANDARD)), + format!("_lr_env={}", encoded(&json!({ "envelope": 5 }), &STANDARD)), + format!("_lr_env={}", encoded(&json!({ "envelope": "" }), &STANDARD)), + format!( + "_lr_env={}", + encoded(&json!({ "envelope": "0" }), &STANDARD) + ), + ] { + assert!( + read(&*provider, &header).is_none(), + "should ignore {header:?}" + ); + } + } + + #[test] + fn requires_exactly_one_source_and_a_cookie_name() { + for (sources, settings) in [ + (json!([]), json!({})), + (json!(["a.example.com", "b.example.com"]), json!({})), + (json!(["a.example.com"]), json!({ "cookie_names": [] })), + ] { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "liveramp", + "kind": "liveramp", + "sources": sources, + "settings": settings, + })) + .expect("should parse provider config"); + + let error = build_kind::(&config, &ProviderSecrets::default()) + .err() + .expect("should reject config"); + + assert!( + matches!(error.current_context(), ProviderConfigError::Invalid), + "should report invalid config" + ); + } + } + + #[test] + fn makes_no_lookup_without_an_endpoint() { + let provider = provider(&json!({})); + let consent = ConsentContext::default(); + let lookup = LookupInput { + ec_id: "ec-example", + consent: &consent, + client: ClientSignals { + ip: None, + user_agent: None, + }, + publisher_domain: "www.example.com", + due_sources: &[], + current: OwnedEntries::empty(), + cookies: RequestCookies::default(), + hashed_email: None, + now: trusted_server_core::ec::eid_types::UnixSeconds::new(1_700_000_000), + }; + + assert!( + provider.build_request(&lookup).is_err(), + "should build no request" + ); + } + + fn lookup_provider() -> Box { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "liveramp", + "kind": "liveramp", + "endpoint": "https://envelope.example.com/api/identity/v2/envelope", + "sources": ["liveramp.example.com"], + "settings": {"placement_id": 14, "custom_id_namespace": "ana-example"}, + })) + .expect("should parse provider config"); + build_kind::(&config, &ProviderSecrets::default()).expect("should build provider") + } + + fn input<'a>(consent: &'a ConsentContext, ip: Option<&str>) -> LookupInput<'a> { + LookupInput { + ec_id: "abc123.suffix", + consent, + client: ClientSignals { + ip: ip.map(|ip| ip.parse().expect("should parse ip")), + user_agent: None, + }, + publisher_domain: "www.example.com", + due_sources: &[], + current: OwnedEntries::empty(), + cookies: RequestCookies::default(), + hashed_email: None, + now: trusted_server_core::ec::eid_types::UnixSeconds::new(1_700_000_000), + } + } + + fn query(request: &ProviderRequest) -> Vec<(String, String)> { + request + .url + .query_pairs() + .map(|(key, value)| (key.into_owned(), value.into_owned())) + .collect() + } + + fn pairs(expected: &[(&str, &str)]) -> Vec<(String, String)> { + expected + .iter() + .map(|(key, value)| ((*key).to_owned(), (*value).to_owned())) + .collect() + } + + #[test] + fn looks_up_an_envelope_by_custom_id() { + let provider = lookup_provider(); + let consent = ConsentContext { + raw_tc_string: Some("CP_example".to_owned()), + raw_gpp_string: Some("DBA_example".to_owned()), + gpp_section_ids: Some(vec![2, 7]), + ..ConsentContext::default() + }; + + let request = provider + .build_request(&input(&consent, Some("11.22.33.44"))) + .expect("should build request"); + + assert_eq!(request.method, Method::GET, "should get"); + assert_eq!( + request.url.path(), + "/api/identity/v2/envelope", + "should call the configured endpoint" + ); + assert_eq!( + query(&request), + pairs(&[ + ("pid", "14"), + ("it", "15"), + ("iv", "ana-example:abc123"), + ("atype", "3"), + ("ct", "4"), + ("cv", "CP_example"), + ("gpp", "DBA_example"), + ("gpp_sid", "2"), + ]), + "should send the stable EC ID as the Custom ID with consent" + ); + assert_eq!( + request.headers.get(ORIGIN).map(HeaderValue::as_bytes), + Some(&b"https://www.example.com"[..]), + "should send the publisher origin" + ); + assert_eq!( + request + .headers + .get("x-forwarded-for") + .map(HeaderValue::as_bytes), + Some(&b"11.22.33.44"[..]), + "should forward the visitor IP" + ); + } + + #[test] + fn sends_us_privacy_when_there_is_no_tcf_string() { + let provider = lookup_provider(); + let consent = ConsentContext { + raw_us_privacy: Some("1YNN".to_owned()), + ..ConsentContext::default() + }; + + let request = provider + .build_request(&input(&consent, Some("11.22.33.44"))) + .expect("should build request"); + + let query = query(&request); + assert!( + query.contains(&("ct".to_owned(), "3".to_owned())) + && query.contains(&("cv".to_owned(), "1YNN".to_owned())), + "should send the US Privacy string" + ); + } + + #[test] + fn needs_a_routable_client_ip() { + let provider = lookup_provider(); + let consent = ConsentContext::default(); + + for ip in [None, Some("10.0.0.1")] { + assert!( + provider.build_request(&input(&consent, ip)).is_err(), + "should not look up without a routable IP" + ); + } + } + + fn reply(status: StatusCode, body: &[u8]) -> Result, Report> { + static HEADERS: std::sync::LazyLock = std::sync::LazyLock::new(HeaderMap::new); + lookup_provider().parse_response(ProviderResponse { + status, + headers: &HEADERS, + body, + context: None, + }) + } + + #[test] + fn stores_the_returned_envelope() { + let body = json!({"envelopes": [ + {"type": 25, "source": "pairIds", "value": "pair-example", "err": null}, + {"type": 19, "source": "envelopeLiveramp", "value": "AhIdEnvelope", "err": null}, + ]}) + .to_string(); + + let outcome = reply(StatusCode::OK, body.as_bytes()).expect("should parse reply"); + + let LookupOutcome::Found(eids) = outcome else { + panic!("should find an envelope"); + }; + assert_eq!( + eids[0].source, "liveramp.example.com", + "should use the source" + ); + assert_eq!( + eids[0].uids[0].id, "AhIdEnvelope", + "should store the envelope" + ); + assert_eq!(eids[0].uids[0].atype, Some(3), "should be person-based"); + assert!( + matches!(eids[0].uids[0].expiry, ProviderExpiry::After(ttl) if ttl == ENVELOPE_TTL), + "should keep the envelope for the envelope TTL" + ); + } + + #[test] + fn maps_no_content_to_an_opt_out_and_a_blank_envelope_to_no_match() { + let blank = json!({"envelopes": [{"type": 19, "value": "", "err": null}]}).to_string(); + let other = json!({"envelopes": [{"type": 25, "value": "pair-example"}]}).to_string(); + let untyped = json!({"envelopes": [ + {"value": "untyped-example"}, + {"type": 19, "value": "rampid-example"}, + ]}) + .to_string(); + + assert!( + matches!( + reply(StatusCode::OK, untyped.as_bytes()), + Ok(LookupOutcome::Found(eids)) if eids[0].uids[0].id == "rampid-example" + ), + "should skip an untyped entry and keep the RampID envelope" + ); + + assert!( + matches!( + reply(StatusCode::OK, other.as_bytes()), + Ok(LookupOutcome::NoMatch) + ), + "should report no match without a RampID envelope" + ); + assert!( + matches!( + reply(StatusCode::NO_CONTENT, b""), + Ok(LookupOutcome::OptedOut) + ), + "should report an opt-out" + ); + assert!( + matches!( + reply(StatusCode::OK, blank.as_bytes()), + Ok(LookupOutcome::NoMatch) + ), + "should report no match" + ); + } + + #[test] + fn rejects_an_error_status_and_a_malformed_body() { + for (status, body) in [ + (StatusCode::UNAVAILABLE_FOR_LEGAL_REASONS, &b"{}"[..]), + (StatusCode::UNAUTHORIZED, &b""[..]), + (StatusCode::OK, &b"not json"[..]), + ] { + assert!(reply(status, body).is_err(), "should reject {status}"); + } + } + + #[test] + fn an_endpoint_needs_a_placement_and_namespace() { + for settings in [ + json!({"placement_id": 14}), + json!({"custom_id_namespace": "ana-example"}), + json!({"placement_id": 14, "custom_id_namespace": " "}), + ] { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "liveramp", + "kind": "liveramp", + "endpoint": "https://envelope.example.com/api/identity/v2/envelope", + "sources": ["liveramp.example.com"], + "settings": settings, + })) + .expect("should parse provider config"); + + assert!( + build_kind::(&config, &ProviderSecrets::default()).is_err(), + "should reject {settings}" + ); + } + } +} diff --git a/crates/trusted-server-identity-providers/src/sharedid.rs b/crates/trusted-server-identity-providers/src/sharedid.rs new file mode 100644 index 000000000..bb0dc7ca7 --- /dev/null +++ b/crates/trusted-server-identity-providers/src/sharedid.rs @@ -0,0 +1,369 @@ +//! Prebid `SharedID` identity provider: reads the publisher common ID from +//! the first-party cookie Prebid's `sharedIdSystem` writes. +//! +//! Prebid stores the ID as a bare UUID in the cookie named by the +//! publisher's `storage.name`. The provider reads the cookie named in +//! `integrations.prebid.managed_user_ids[].storage.name` for the `sharedId` +//! (or `pubCommonId`) module; without one it reads `_sharedid`, then the +//! legacy `_pubcid`. It is sent as a device-based (agent type 1) EID under +//! `pubcid.org` unless another source is configured. The opt-out cookie +//! `_pubcid_optout` reports an opt-out, which removes a stored `SharedID`. +//! No server call is made. +//! +//! A stored `SharedID` lives as long as the row and wins over a different +//! cookie value, so the ID bidders see stays stable. +//! +//! `SharedID` is a publisher first-party ID with no vendor server. With +//! `generate = true`, when the cookies hold none and no live `SharedID` is +//! stored, Trusted Server generates one itself: a random UUID v4, stored in +//! the row only. No cookie is set, so Prebid's own cookie is never touched. +//! An expired generated ID an older row still holds is renewed with the same +//! value. +//! Generation is off by default: Prebid may keep its opt-out in localStorage, +//! which the server cannot see, so only a publisher who knows its opt-out +//! lives in the `_pubcid_optout` cookie should turn it on. + +use std::borrow::Cow; + +use error_stack::Report; +use serde::Deserialize; +use trusted_server_core::ec::eid_types::SourceDomain; +use trusted_server_core::ec::provider::{ + CookieRead, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, + ProviderConfigError, ProviderEid, ProviderError, ProviderExpiry, ProviderKind, ProviderRequest, + ProviderResponse, RequestCookies, +}; +use uuid::Uuid; + +use crate::common::{CookieReader, cookie_capabilities, device_eid, is_usable_id}; + +/// Settings key naming the cookies that hold the `SharedID`. +pub const COOKIE_NAMES_SETTING: &str = "cookie_names"; + +/// Cookie Prebid sets when the user opts out of `SharedID`. +const OPT_OUT_COOKIE: &str = "_pubcid_optout"; + +/// The Prebid `SharedID` provider kind. +pub struct SharedId; + +/// Settings for a `sharedid` provider. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SharedIdSettings { + /// Cookies holding the `SharedID`, read in order. Defaults to Prebid's + /// configured `storage.name`, else `_sharedid` then `_pubcid`. + #[serde(default = "SharedIdSettings::default_cookie_names")] + pub cookie_names: Vec, + /// Whether to generate a `SharedID` when the cookies hold none. Off by + /// default, because a localStorage opt-out is invisible to the server. + #[serde(default)] + pub generate: bool, +} + +impl SharedIdSettings { + fn default_cookie_names() -> Vec { + vec!["_sharedid".to_owned(), "_pubcid".to_owned()] + } +} + +/// A built Prebid `SharedID` provider. +#[derive(Debug)] +pub struct SharedIdProvider { + capabilities: ProviderCapabilities, + source: SourceDomain, + cookies: CookieReader, + generate: bool, +} + +impl ProviderKind for SharedId { + const KIND: &'static str = "sharedid"; + type Settings = SharedIdSettings; + type Provider = SharedIdProvider; + + fn build( + common: &ProviderCommon<'_>, + settings: SharedIdSettings, + ) -> Result> { + if settings.cookie_names.is_empty() { + return Err(Report::new(ProviderConfigError::Invalid) + .attach("a sharedid provider needs at least one cookie name")); + } + let (capabilities, source) = cookie_capabilities(common, Self::KIND)?; + Ok(SharedIdProvider { + capabilities, + source, + cookies: CookieReader::own(common, settings.cookie_names), + generate: settings.generate, + }) + } +} + +impl IdentityProvider for SharedIdProvider { + fn capabilities(&self) -> &ProviderCapabilities { + &self.capabilities + } + + fn build_request( + &self, + _lookup: &LookupInput<'_>, + ) -> Result> { + Err(Report::new(ProviderError::Request).attach("sharedid makes no lookup")) + } + + fn parse_response<'body>( + &self, + _response: ProviderResponse<'body>, + ) -> Result, Report> { + Err(Report::new(ProviderError::InvalidResponse).attach("sharedid makes no lookup")) + } + + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> CookieRead<'cookie> { + if cookies.get(OPT_OUT_COOKIE).is_some() { + return CookieRead::OptedOut; + } + let Some(id) = self + .cookies + .read(cookies, |value| is_usable_id(value).then_some(value.trim())) + else { + return CookieRead::Absent; + }; + // No expiry: the ID lives as long as the row. + CookieRead::Found(vec![device_eid( + &self.source, + Cow::Borrowed(id), + ProviderExpiry::Unspecified, + )]) + } + + fn keeps_stored_uids(&self) -> bool { + true + } + + fn generate(&self, lookup: &LookupInput<'_>) -> Option>> { + if !self.generate { + return None; + } + // An expired ID the row still holds is renewed rather than replaced, + // so the generated ID stays stable for as long as the row lives. + let id = lookup + .current + .iter() + .filter(|(source, _)| *source == self.source.as_str()) + .flat_map(|(_, eid)| eid.uids()) + .map(|uid| uid.id().as_str().to_owned()) + .next() + .unwrap_or_else(|| Uuid::new_v4().to_string()); + Some(vec![device_eid( + &self.source, + Cow::Owned(id), + // A generated ID lives as long as the row. + ProviderExpiry::Unspecified, + )]) + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + use trusted_server_core::ec::eid_types::{ + AgentType, EidFields, Expiry, ProviderId, UidValue, UnixSeconds, + }; + use trusted_server_core::ec::kv_types::{KvEid, KvEntry, KvPartnerId, KvUid}; + use trusted_server_core::ec::provider::{OwnedEntries, build_kind}; + use trusted_server_core::settings::{IdentityProviderConfig, ProviderSecrets}; + + use super::*; + + fn provider() -> Box { + provider_with(&json!({})) + } + + fn provider_with(settings: &serde_json::Value) -> Box { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "sharedid", + "kind": "sharedid", + "sources": ["sharedid.example.com"], + "settings": settings, + })) + .expect("should parse provider config"); + build_kind::(&config, &ProviderSecrets::default()).expect("should build provider") + } + + fn read(header: &str) -> Option { + let eids = provider() + .read_request_cookies(RequestCookies::new(header)) + .into_found()?; + assert_eq!( + eids[0].source, "sharedid.example.com", + "should use the source" + ); + let uid = &eids[0].uids[0]; + assert_eq!(uid.atype, Some(1), "should be device-based"); + Some(uid.id.to_string()) + } + + #[test] + fn reads_sharedid_before_pubcid() { + assert_eq!( + read("_pubcid=pubcid-example; _sharedid=sharedid-example").as_deref(), + Some("sharedid-example"), + "should prefer _sharedid" + ); + assert_eq!( + read("_pubcid=pubcid-example").as_deref(), + Some("pubcid-example"), + "should fall back to _pubcid" + ); + } + + #[test] + fn keeps_a_cookie_id_as_long_as_the_row_and_over_a_different_cookie() { + let provider = provider(); + let eids = provider + .read_request_cookies(RequestCookies::new("_sharedid=sharedid-example")) + .into_found() + .expect("should read the cookie"); + + assert!( + matches!(eids[0].uids[0].expiry, ProviderExpiry::Unspecified), + "should state no expiry, so the ID lives as long as the row" + ); + assert!( + provider.keeps_stored_uids(), + "should keep a stored ID over a different cookie" + ); + } + + #[test] + fn defaults_the_source_to_pubcid_org() { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "sharedid", + "kind": "sharedid", + })) + .expect("should parse provider config"); + + let provider = build_kind::(&config, &ProviderSecrets::default()) + .expect("should build provider"); + let eids = provider + .read_request_cookies(RequestCookies::new("_sharedid=sharedid-example")) + .into_found() + .expect("should read the cookie"); + + assert_eq!(eids[0].source, "pubcid.org", "should default to pubcid.org"); + assert_eq!(eids[0].uids[0].atype, Some(1), "should be device-based"); + } + + #[test] + fn honours_opt_out_and_ignores_blank_ids() { + assert!( + matches!( + provider().read_request_cookies(RequestCookies::new( + "_sharedid=sharedid-example; _pubcid_optout=1" + )), + CookieRead::OptedOut + ), + "should report the opt-out cookie" + ); + for header in ["_sharedid=", "_sharedid=0", "other=1"] { + assert!(read(header).is_none(), "should ignore {header}"); + } + } + + fn generate_with(provider: &dyn IdentityProvider) -> Option>> { + generate_holding(provider, OwnedEntries::empty()) + } + + fn generate_holding( + provider: &dyn IdentityProvider, + current: OwnedEntries<'_>, + ) -> Option>> { + let consent = trusted_server_core::consent::ConsentContext::default(); + let lookup = LookupInput { + ec_id: "ec-example", + consent: &consent, + client: trusted_server_core::ec::provider::ClientSignals { + ip: None, + user_agent: None, + }, + publisher_domain: "www.example.com", + due_sources: &[], + current, + cookies: RequestCookies::default(), + hashed_email: None, + now: trusted_server_core::ec::eid_types::UnixSeconds::new(1_700_000_000), + }; + provider.generate(&lookup) + } + + #[test] + fn generates_nothing_by_default() { + assert!( + generate_with(&*provider()).is_none(), + "should not generate without generate = true" + ); + } + + #[test] + fn renews_an_expired_generated_id_the_row_still_holds() { + // Arrange: the row holds the generated ID, expired at 1000. + let provider = provider_with(&json!({ "generate": true })); + let uid = KvUid::new( + UidValue::try_from("11111111-2222-4333-8444-555555555555").expect("should build uid"), + AgentType::try_from(1).expect("should build atype"), + None, + Some(ProviderId::new("sharedid")), + Expiry::At(UnixSeconds::new(1000)), + ); + let mut entry = KvEntry::tombstone(1); + entry.ids.insert( + "sharedid.example.com".to_owned(), + KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![uid])], + UnixSeconds::new(500), + ) + .expect("should build record"), + ); + let sources = [SourceDomain::try_from("sharedid.example.com").expect("should parse")]; + let owner = ProviderId::new("sharedid"); + + // Act + let eids = generate_holding( + &*provider, + OwnedEntries::new(Some(&entry), &sources, &owner), + ) + .expect("should generate an ID"); + + // Assert + assert_eq!( + eids[0].uids[0].id, "11111111-2222-4333-8444-555555555555", + "should keep the held ID rather than mint a new one" + ); + } + + #[test] + fn generates_a_random_uuid_when_none_is_held() { + let provider = provider_with(&json!({ "generate": true })); + + let first = generate_with(&*provider).expect("should generate an ID"); + let second = generate_with(&*provider).expect("should generate an ID"); + + let uid = &first[0].uids[0]; + assert_eq!( + first[0].source, "sharedid.example.com", + "should use the source" + ); + assert_eq!(uid.atype, Some(1), "should be device-based"); + assert!( + Uuid::parse_str(&uid.id).is_ok_and(|id| id.get_version_num() == 4), + "should be a UUID v4" + ); + assert_ne!(uid.id, second[0].uids[0].id, "should be random"); + assert!( + matches!(uid.expiry, ProviderExpiry::Unspecified), + "should keep a generated ID as long as the row" + ); + } +} diff --git a/crates/trusted-server-identity-providers/src/uid2.rs b/crates/trusted-server-identity-providers/src/uid2.rs new file mode 100644 index 000000000..2e4362182 --- /dev/null +++ b/crates/trusted-server-identity-providers/src/uid2.rs @@ -0,0 +1,973 @@ +//! UID2 identity provider: reads the UID2 advertising token from the +//! browser's own first-party cookies, and refreshes it server side when an +//! `endpoint` is configured. +//! +//! The UID2 SDK stores its identity in the `__uid_2` cookie as +//! percent-encoded JSON. `advertising_token` and `identity_expires` +//! (milliseconds) are read: the token is stored with agent type 3 until +//! `identity_expires`, and the coordinator drops it once that time has +//! passed. +//! +//! With an `endpoint` (UID2's `POST /v2/token/refresh`), the cookie's +//! `refresh_token` and `refresh_response_key` are also kept with the token as +//! private refresh state, and the source becomes due at `refresh_from`. The +//! refresh posts the refresh token, which needs no API key and no email, and +//! decrypts the reply with the response key (AES-256-GCM, a 12-byte IV +//! first). A refresh replying `optout` removes the stored token. No token is +//! ever generated: that needs an email or phone number, which is out of +//! scope. Without an `endpoint` the refresh state is not stored. +//! +//! When `__uid_2` yields nothing, the cookies in `cookie_names` (default +//! Prebid's `__uid2_advertising_token`) are read. Prebid's UID2 module +//! stores percent-encoded JSON there, `{"originalToken":…,"latestToken":…}`: +//! only `latestToken.advertising_token` is stored, until +//! `latestToken.identity_expires` (milliseconds) or with the shared default +//! TTL when that is missing. Any other value is read as a raw token with the +//! shared default TTL. With the Lockr capability enabled +//! (`[settings.lockr]`), the cookies Lockr AIM writes the raw token into +//! (default key `__uid2_advertising_token`) are read last, with the same +//! parser. +//! +//! A `{"status":"optout"}` identity in `__uid_2` (the UID2 SDK's opt-out) +//! or a `latestToken` of `"optout"` in any of those cookies reports an +//! opt-out: no lookup is made and stored UID2 UIDs are removed. +//! +//! UID2 is not offered where GDPR applies: there the provider is skipped +//! for region, so its cookies are not read, no refresh is made and the UIDs +//! the row holds for it are removed. +//! +//! Parsing is lenient: an undecodable cookie, a missing token or expiry, +//! or a blank or placeholder (`0`) token reads as no ID. A JSON value is never stored +//! as a token. + +use std::borrow::Cow; + +use aes_gcm::aead::{Aead as _, KeyInit as _}; +use aes_gcm::{Aes256Gcm, Nonce}; +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; +use bytes::Bytes; +use error_stack::Report; +use http::header::{CONTENT_TYPE, HeaderMap, HeaderValue}; +use serde::{Deserialize, Serialize}; +use trusted_server_core::ec::eid_types::{SourceDomain, UnixSeconds}; +use trusted_server_core::ec::provider::{ + CookieRead, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, ProviderCommon, + ProviderConfigError, ProviderEid, ProviderError, ProviderExpiry, ProviderKind, ProviderRefresh, + ProviderRequest, ProviderResponse, RequestCookies, +}; +use url::Url; + +use crate::common::{ + CookieReader, LockrSettings, ReadsLockr, bounded_expiry, cookie_capabilities, cookie_eid, + is_expired, is_usable_id, percent_decoded, +}; + +/// The UID2 provider kind. +pub struct Uid2; + +/// Settings for a `uid2` provider. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Uid2Settings { + /// Cookie holding the UID2 SDK identity JSON. + #[serde(default = "Uid2Settings::default_identity_cookie_name")] + pub identity_cookie_name: String, + /// Cookies holding a raw advertising token, read in order after the + /// identity cookie. + #[serde(default = "Uid2Settings::default_cookie_names")] + pub cookie_names: Vec, + /// The Lockr capability; see [`LockrSettings`]. + #[serde(default)] + pub lockr: LockrSettings, +} + +impl Uid2Settings { + fn default_identity_cookie_name() -> String { + "__uid_2".to_owned() + } + + fn default_cookie_names() -> Vec { + vec!["__uid2_advertising_token".to_owned()] + } +} + +/// A built UID2 provider. +#[derive(Debug)] +pub struct Uid2Provider { + capabilities: ProviderCapabilities, + source: SourceDomain, + identity_cookie_name: String, + cookies: CookieReader, + /// UID2's token refresh endpoint; `None` disables server refresh. + refresh_endpoint: Option, +} + +impl ReadsLockr for Uid2 { + const LOCKR_COOKIE_NAMES: &'static [&'static str] = &["__uid2_advertising_token"]; +} + +impl ProviderKind for Uid2 { + const KIND: &'static str = "uid2"; + type Settings = Uid2Settings; + type Provider = Uid2Provider; + + fn build( + common: &ProviderCommon<'_>, + settings: Uid2Settings, + ) -> Result> { + let (capabilities, source) = cookie_capabilities(common, Self::KIND)?; + Ok(Uid2Provider { + capabilities, + source, + identity_cookie_name: settings.identity_cookie_name, + cookies: CookieReader::with_lockr::( + common, + settings.cookie_names, + settings.lockr, + )?, + refresh_endpoint: common.endpoint.cloned(), + }) + } +} + +/// A UID2 identity: the `__uid_2` cookie and the `body` of a decrypted +/// refresh reply share these fields. +#[derive(Deserialize)] +struct Identity { + advertising_token: String, + identity_expires: u64, + #[serde(default)] + refresh_token: Option, + #[serde(default)] + refresh_response_key: Option, + #[serde(default)] + refresh_from: Option, +} + +/// The refresh state kept with a stored token. +#[derive(Serialize, Deserialize)] +struct RefreshState { + /// The refresh token posted to the refresh endpoint. + token: String, + /// Base64 AES-256 key that decrypts the refresh reply. + key: String, +} + +/// A usable token with its expiry and, when present, its refresh state. +struct Token { + token: String, + expiry: ProviderExpiry, + refresh: Option<(RefreshState, ProviderExpiry)>, +} + +impl Identity { + /// The token, with refresh state when the identity carries all of it. + fn into_token(self) -> Option { + if !is_usable_id(&self.advertising_token) { + return None; + } + let refresh = match ( + self.refresh_token, + self.refresh_response_key, + self.refresh_from, + ) { + (Some(token), Some(key), Some(from)) if !token.is_empty() && !key.is_empty() => Some(( + RefreshState { token, key }, + ProviderExpiry::At(UnixSeconds::new(from / 1000)), + )), + _ => None, + }; + Some(Token { + token: self.advertising_token, + expiry: ProviderExpiry::At(UnixSeconds::new(self.identity_expires / 1000)), + refresh, + }) + } +} + +/// Decodes one `__uid_2` value. +fn identity(value: &str) -> Option { + let decoded = percent_decoded(value); + serde_json::from_str::(&decoded) + .ok()? + .into_token() +} + +/// A decrypted refresh reply. +#[derive(Deserialize)] +struct RefreshReply { + status: String, + #[serde(default)] + body: Option, +} + +/// Status of a refresh reply for a user who opted out. +const REFRESH_OPT_OUT: &str = "optout"; + +/// Length of the AES-GCM IV that starts a refresh reply. +const IV_BYTES: usize = 12; + +/// Decrypts a base64 refresh reply with the base64 `key`. +fn decrypt_reply(body: &[u8], key: &str) -> Option> { + let key = BASE64.decode(key).ok()?; + let cipher = Aes256Gcm::new_from_slice(&key).ok()?; + let envelope = BASE64.decode(body.trim_ascii()).ok()?; + if envelope.len() <= IV_BYTES { + return None; + } + let (iv, ciphertext) = envelope.split_at(IV_BYTES); + let nonce = Nonce::try_from(iv).ok()?; + cipher.decrypt(&nonce, ciphertext).ok() +} + +/// `latestToken` value Prebid stores after an opt-out. +const PREBID_OPT_OUT: &str = "optout"; + +/// The fields of Prebid's stored tokens this provider reads. +#[derive(Deserialize)] +struct PrebidTokens { + #[serde(rename = "latestToken")] + latest_token: serde_json::Value, +} + +/// The fields of Prebid's `latestToken` identity this provider reads. +#[derive(Deserialize)] +struct PrebidIdentity { + advertising_token: String, + #[serde(default)] + identity_expires: Option, +} + +/// One token cookie's reading. +enum TokenCookie { + /// A usable token and its expiry. + Token(String, ProviderExpiry), + /// Prebid recorded an opt-out. + OptedOut, +} + +/// Decodes one token cookie: Prebid's stored tokens JSON or a raw token. +/// +/// A JSON value is never taken as a raw token. +fn token_cookie(value: &str) -> Option { + let decoded = percent_decoded(value); + let decoded = decoded.trim(); + if decoded.starts_with('{') { + return prebid_tokens(decoded); + } + is_usable_id(decoded).then(|| TokenCookie::Token(decoded.to_owned(), bounded_expiry(None))) +} + +/// Reads `latestToken` from Prebid's stored tokens JSON. +fn prebid_tokens(json: &str) -> Option { + let tokens: PrebidTokens = serde_json::from_str(json).ok()?; + if tokens.latest_token.as_str() == Some(PREBID_OPT_OUT) { + return Some(TokenCookie::OptedOut); + } + let identity: PrebidIdentity = serde_json::from_value(tokens.latest_token).ok()?; + if !is_usable_id(&identity.advertising_token) { + return None; + } + let expiry = identity.identity_expires.map_or_else( + || bounded_expiry(None), + |ms| ProviderExpiry::At(UnixSeconds::new(ms / 1000)), + ); + Some(TokenCookie::Token(identity.advertising_token, expiry)) +} + +/// Whether a `uid2` provider excludes `source` where GDPR applies: UID2 is +/// not offered there, so every source is excluded. +pub(crate) fn excludes_where_gdpr_applies(_source: &str) -> bool { + true +} + +impl IdentityProvider for Uid2Provider { + fn capabilities(&self) -> &ProviderCapabilities { + &self.capabilities + } + + /// UID2 is not offered where GDPR applies, so none of its sources is + /// collected or kept there. + fn excluded_where_gdpr_applies(&self, source: &SourceDomain) -> bool { + excludes_where_gdpr_applies(source.as_str()) + } + + /// A server-side refresh does not rewrite the browser's cookies, so a + /// token stored by a refresh outlives the cookie's older token and is + /// kept until the browser SDK refreshes to a newer one. + fn keeps_fresher_stored_uids(&self) -> bool { + true + } + + fn build_request( + &self, + lookup: &LookupInput<'_>, + ) -> Result> { + let Some(endpoint) = &self.refresh_endpoint else { + return Err(Report::new(ProviderError::Request) + .attach("uid2 refreshes only with an endpoint and never generates tokens")); + }; + let state = lookup + .current + .iter() + .flat_map(|(_, eid)| eid.uids()) + .find_map(|uid| uid.refresh()) + .and_then(|refresh| serde_json::from_str::(&refresh.state).ok()) + .ok_or_else(|| { + Report::new(ProviderError::Request).attach("no stored uid2 refresh token") + })?; + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_TYPE, HeaderValue::from_static("text/plain")); + Ok(ProviderRequest { + method: http::Method::POST, + url: endpoint.clone(), + headers, + body: Bytes::from(state.token), + context: Some(state.key), + }) + } + + fn parse_response<'body>( + &self, + response: ProviderResponse<'body>, + ) -> Result, Report> { + if response.status == http::StatusCode::NO_CONTENT { + return Ok(LookupOutcome::NoMatch); + } + let key = response.context.ok_or_else(|| { + Report::new(ProviderError::InvalidResponse).attach("no uid2 response key") + })?; + let plaintext = decrypt_reply(response.body, key).ok_or_else(|| { + Report::new(ProviderError::InvalidResponse).attach("uid2 reply not decrypted") + })?; + let reply: RefreshReply = serde_json::from_slice(&plaintext).map_err(|error| { + Report::new(ProviderError::InvalidResponse).attach(format!("uid2 reply: {error}")) + })?; + if reply.status == REFRESH_OPT_OUT { + return Ok(LookupOutcome::OptedOut); + } + let token = reply + .body + .filter(|_| reply.status == "success") + .and_then(Identity::into_token) + .ok_or_else(|| { + Report::new(ProviderError::InvalidResponse) + .attach(format!("uid2 refresh status {}", reply.status)) + })?; + Ok(LookupOutcome::Found(vec![self.eid(token)])) + } + + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> CookieRead<'cookie> { + // The UID2 SDK records an opt-out as `{"status":"optout"}` in its + // identity cookie, Prebid as a `latestToken` of `"optout"`; either + // one, in any duplicate cookie, wins over every token. + let opted_out = cookies.get_all(&self.identity_cookie_name).any(|value| { + serde_json::from_str::(&percent_decoded(value)) + .is_ok_and(|identity| identity["status"] == REFRESH_OPT_OUT) + }) || self.cookies.any(cookies, |value| { + matches!(token_cookie(value), Some(TokenCookie::OptedOut)) + }); + if opted_out { + return CookieRead::OptedOut; + } + // An expired identity never hides a later live one. + let token = cookies + .get_all(&self.identity_cookie_name) + .filter_map(identity) + .find(|token| !is_expired(&token.expiry, cookies.now())) + .or_else(|| { + self.cookies + .read(cookies, |value| match token_cookie(value)? { + TokenCookie::Token(token, expiry) + if !is_expired(&expiry, cookies.now()) => + { + Some(Token { + token, + expiry, + refresh: None, + }) + } + TokenCookie::Token(..) | TokenCookie::OptedOut => None, + }) + }); + let Some(token) = token else { + return CookieRead::Absent; + }; + CookieRead::Found(vec![self.eid(token)]) + } +} + +impl Uid2Provider { + /// Builds the entry for `token`, keeping its refresh state only when + /// server refresh is enabled. + fn eid<'any>(&self, token: Token) -> ProviderEid<'any> { + let mut eid = cookie_eid(&self.source, Cow::Owned(token.token), token.expiry); + if self.refresh_endpoint.is_some() + && let Some((state, from)) = token.refresh + && let Ok(state) = serde_json::to_string(&state) + { + for uid in &mut eid.uids { + uid.refresh = Some(ProviderRefresh { + state: Cow::Owned(state.clone()), + from, + }); + } + } + eid + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + use trusted_server_core::ec::provider::build_kind; + use trusted_server_core::settings::{IdentityProviderConfig, ProviderSecrets}; + + use trusted_server_core::consent::ConsentContext; + use trusted_server_core::ec::eid_types::{AgentType, EidFields, Expiry, ProviderId, UidValue}; + use trusted_server_core::ec::kv_types::{KvEid, KvEntry, KvPartnerId, KvRefresh, KvUid}; + use trusted_server_core::ec::provider::{ClientSignals, OwnedEntries}; + + use super::*; + + /// Request time the tests read cookies at. + const NOW: UnixSeconds = UnixSeconds::new(1_700_000_000); + + fn provider(settings: &serde_json::Value) -> Box { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "uid2", + "kind": "uid2", + "sources": ["uid2.example.com"], + "settings": settings, + })) + .expect("should parse provider config"); + build_kind::(&config, &ProviderSecrets::default()).expect("should build provider") + } + + fn read(provider: &dyn IdentityProvider, header: &str) -> Option<(String, ProviderExpiry)> { + let eids = provider + .read_request_cookies(RequestCookies::new(header).at(NOW)) + .into_found()?; + assert_eq!(eids[0].source, "uid2.example.com", "should use the source"); + let uid = &eids[0].uids[0]; + assert_eq!(uid.atype, Some(3), "should be person-based"); + Some((uid.id.to_string(), uid.expiry)) + } + + fn identity_cookie(identity: &serde_json::Value) -> String { + format!("__uid_2={}", urlencoding::encode(&identity.to_string())) + } + + #[test] + fn is_not_run_where_gdpr_applies() { + let provider = provider(&json!({})); + let source = SourceDomain::try_from("uid2.example.com").expect("should build source"); + + assert!( + provider.excluded_where_gdpr_applies(&source), + "should not run UID2 where GDPR applies" + ); + } + + #[test] + fn reads_only_the_advertising_token_until_identity_expiry() { + let header = identity_cookie(&json!({ + "advertising_token": "AgAAAAexampletoken+/=", + "refresh_token": "example-refresh-token", + "identity_expires": 1_800_000_000_123_u64, + "refresh_from": 4_699_990_000_000_u64, + "refresh_expires": 4_702_000_000_000_u64, + "refresh_response_key": "example-response-key", + })); + + let (token, expiry) = read(&*provider(&json!({})), &header).expect("should read"); + + assert_eq!( + token, "AgAAAAexampletoken+/=", + "should store only the token" + ); + assert!( + matches!(expiry, ProviderExpiry::At(at) if at.get() == 1_800_000_000), + "should expire at identity_expires" + ); + } + + #[test] + fn falls_back_to_the_raw_token_cookie() { + let header = "__uid_2=%7Bbroken; __uid2_advertising_token=AgAAAAexampletoken"; + + let (token, expiry) = read(&*provider(&json!({})), header).expect("should read"); + + assert_eq!(token, "AgAAAAexampletoken", "should read the raw token"); + assert!( + matches!(expiry, ProviderExpiry::After(_)), + "should use the default TTL" + ); + } + + fn prebid_cookie(tokens: &serde_json::Value) -> String { + format!( + "__uid2_advertising_token={}", + urlencoding::encode(&tokens.to_string()) + ) + } + + #[test] + fn reads_the_latest_token_from_prebid_stored_tokens() { + let header = prebid_cookie(&json!({ + "originalToken": { + "advertising_token": "AgAAAAoriginaltoken", + "identity_expires": 1_600_000_000_000_u64, + }, + "latestToken": { + "advertising_token": "AgAAAAlatesttoken", + "refresh_token": "example-refresh-token", + "refresh_response_key": "example-response-key", + "identity_expires": 1_800_000_000_456_u64, + "refresh_from": 4_699_990_000_000_u64, + "refresh_expires": 4_702_000_000_000_u64, + }, + })); + + let (token, expiry) = read(&*provider(&json!({})), &header).expect("should read"); + + assert_eq!( + token, "AgAAAAlatesttoken", + "should store only the latest advertising token" + ); + assert!( + matches!(expiry, ProviderExpiry::At(at) if at.get() == 1_800_000_000), + "should expire at the latest identity_expires" + ); + } + + #[test] + fn reports_a_prebid_opt_out() { + let provider = provider(&json!({})); + let own = identity_cookie(&json!({ + "advertising_token": "AgAAAAowntoken", + "identity_expires": 1_800_000_000_000_u64, + })); + let opt_out = prebid_cookie(&json!({ + "originalToken": { "advertising_token": "AgAAAAoriginaltoken" }, + "latestToken": "optout", + })); + + let header = format!("{own}; {opt_out}"); + + let read = provider.read_request_cookies(RequestCookies::new(&header).at(NOW)); + + assert!( + matches!(read, CookieRead::OptedOut), + "should report the opt-out over any token" + ); + } + + #[test] + fn reports_an_sdk_opt_out_over_a_fallback_token() { + let provider = provider(&json!({})); + let live = identity_cookie(&json!({ + "advertising_token": "AgAAAAowntoken", + "identity_expires": 1_800_000_000_000_u64, + })); + let opt_out = identity_cookie(&json!({ "status": "optout" })); + let fallback = prebid_cookie(&json!({ + "latestToken": { + "advertising_token": "AgAAAAlatesttoken", + "identity_expires": 1_800_000_000_000_u64, + }, + })); + let header = format!("{live}; {opt_out}; {fallback}"); + + let read = provider.read_request_cookies(RequestCookies::new(&header).at(NOW)); + + assert!( + matches!(read, CookieRead::OptedOut), + "should report the SDK opt-out over every token" + ); + } + + #[test] + fn reports_an_opt_out_in_a_later_duplicate_cookie() { + let provider = provider(&json!({})); + let token = prebid_cookie(&json!({ + "latestToken": { + "advertising_token": "AgAAAAlatesttoken", + "identity_expires": 1_800_000_000_000_u64, + }, + })); + let opt_out = prebid_cookie(&json!({ "latestToken": "optout" })); + let header = format!("{token}; {opt_out}"); + + let read = provider.read_request_cookies(RequestCookies::new(&header).at(NOW)); + + assert!( + matches!(read, CookieRead::OptedOut), + "should see the opt-out in a later cookie of the same name" + ); + } + + #[test] + fn an_expired_identity_does_not_hide_a_later_live_token() { + let provider = provider(&json!({})); + let expired = identity_cookie(&json!({ + "advertising_token": "AgAAAAexpiredtoken", + "identity_expires": 1_000_u64, + })); + let live = prebid_cookie(&json!({ + "latestToken": { + "advertising_token": "AgAAAAlivetoken", + "identity_expires": 1_800_000_000_000_u64, + }, + })); + let header = format!("{expired}; {live}"); + + let read = read(&*provider, &header); + + assert_eq!( + read.map(|(token, _)| token).as_deref(), + Some("AgAAAAlivetoken"), + "should skip the expired identity" + ); + } + + #[test] + fn never_stores_json_as_a_token() { + let provider = provider(&json!({})); + for header in [ + "__uid2_advertising_token=%7Bbroken".to_owned(), + "__uid2_advertising_token={\"latestToken\":{}}".to_owned(), + prebid_cookie(&json!({ "originalToken": {} })), + prebid_cookie(&json!({ "latestToken": { "advertising_token": "" } })), + prebid_cookie(&json!({ "latestToken": 7 })), + ] { + assert!( + matches!( + provider.read_request_cookies(RequestCookies::new(&header).at(NOW)), + CookieRead::Absent + ), + "should read nothing from {header:?}" + ); + } + } + + #[test] + fn reads_configured_cookie_names() { + let provider = provider(&json!({ "cookie_names": ["example_lockr_uid2"] })); + + let read = read(&*provider, "example_lockr_uid2=AgAAAAexampletoken"); + + assert_eq!( + read.map(|(token, _)| token).as_deref(), + Some("AgAAAAexampletoken"), + "should read the configured cookie" + ); + } + + #[test] + fn reads_the_lockr_cookie_last() { + let enabled = provider(&json!({ + "lockr": { "enabled": true, "cookie_names": ["example_lockr_uid2"] }, + })); + let disabled = provider(&json!({ + "lockr": { "cookie_names": ["example_lockr_uid2"] }, + })); + let own = identity_cookie(&json!({ + "advertising_token": "AgAAAAowntoken", + "identity_expires": 1_800_000_000_000_u64, + })); + + let both = read(&*enabled, &format!("example_lockr_uid2=AgAAAAlockr; {own}")); + let lockr_only = read(&*enabled, "example_lockr_uid2=AgAAAAlockr"); + let ignored = read(&*disabled, "example_lockr_uid2=AgAAAAlockr"); + + assert_eq!( + both.map(|(token, _)| token).as_deref(), + Some("AgAAAAowntoken"), + "should prefer the identity cookie" + ); + assert_eq!( + lockr_only.map(|(token, _)| token).as_deref(), + Some("AgAAAAlockr"), + "should read the Lockr cookie as a raw token" + ); + assert!(ignored.is_none(), "should ignore Lockr while disabled"); + } + + #[test] + fn ignores_missing_malformed_and_opt_out_cookies() { + let provider = provider(&json!({})); + for header in [ + String::new(), + identity_cookie(&json!({ "advertising_token": "tok" })), + identity_cookie(&json!({ "identity_expires": 1 })), + identity_cookie(&json!({ "advertising_token": "", "identity_expires": 1 })), + "__uid2_advertising_token=0".to_owned(), + ] { + assert!( + read(&*provider, &header).is_none(), + "should ignore {header:?}" + ); + } + } + + const REFRESH_ENDPOINT: &str = "https://uid2.example.com/v2/token/refresh"; + + fn refreshing_provider() -> Box { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "uid2", + "kind": "uid2", + "sources": ["uid2.example.com"], + "endpoint": REFRESH_ENDPOINT, + "allowed_domains": ["uid2.example.com"], + })) + .expect("should parse provider config"); + build_kind::(&config, &ProviderSecrets::default()).expect("should build provider") + } + + fn sdk_identity_header() -> String { + identity_cookie(&json!({ + "advertising_token": "AgAAAAexampletoken", + "refresh_token": "example-refresh-token", + "identity_expires": 1_800_000_000_000_u64, + "refresh_from": 4_699_990_000_000_u64, + "refresh_expires": 4_702_000_000_000_u64, + "refresh_response_key": BASE64.encode([7_u8; 32]), + })) + } + + fn refresh_of( + provider: &dyn IdentityProvider, + header: &str, + ) -> Option<(String, ProviderExpiry)> { + let eids = provider + .read_request_cookies(RequestCookies::new(header).at(NOW)) + .into_found() + .expect("should read the token"); + let refresh = eids[0].uids[0].refresh.as_ref()?; + Some((refresh.state.to_string(), refresh.from)) + } + + #[test] + fn keeps_the_refresh_state_when_refresh_is_enabled() { + let (state, from) = refresh_of(&*refreshing_provider(), &sdk_identity_header()) + .expect("should keep the refresh state"); + + let state: RefreshState = serde_json::from_str(&state).expect("should parse the state"); + assert_eq!( + state.token, "example-refresh-token", + "should keep the refresh token" + ); + assert!( + matches!(from, ProviderExpiry::At(at) if at.get() == 4_699_990_000), + "should ask for a refresh at refresh_from" + ); + } + + #[test] + fn stores_no_refresh_state_without_an_endpoint() { + assert!( + refresh_of(&*provider(&json!({})), &sdk_identity_header()).is_none(), + "should not keep the refresh token when refresh is disabled" + ); + } + + fn stored_entry() -> KvEntry { + let state = serde_json::to_string(&RefreshState { + token: "example-refresh-token".to_owned(), + key: BASE64.encode([7_u8; 32]), + }) + .expect("should serialize the state"); + let uid = KvUid::new( + UidValue::try_from("AgAAAAexampletoken".to_owned()).expect("should build uid"), + AgentType::try_from(3).expect("should build atype"), + None, + Some(ProviderId::new("uid2")), + Expiry::At(UnixSeconds::new(1_800_000_000)), + ) + .with_refresh(KvRefresh { + state, + from: UnixSeconds::new(1_799_990_000), + }); + let record = KvPartnerId::from_eids( + vec![KvEid::new(EidFields::default(), vec![uid])], + UnixSeconds::new(1_799_995_000), + ) + .expect("should keep the live uid"); + let mut entry = KvEntry::tombstone(1); + entry.consent.ok = true; + entry.ids.insert("uid2.example.com".to_owned(), record); + entry + } + + #[test] + fn refresh_posts_the_stored_refresh_token() { + let entry = stored_entry(); + let sources = [SourceDomain::try_from("uid2.example.com").expect("should build source")]; + let owner = ProviderId::new("uid2"); + let consent = ConsentContext::default(); + let lookup = LookupInput { + ec_id: "ec-example", + consent: &consent, + client: ClientSignals { + ip: None, + user_agent: None, + }, + publisher_domain: "www.example.com", + due_sources: &sources, + current: OwnedEntries::new(Some(&entry), &sources, &owner), + cookies: RequestCookies::default(), + hashed_email: None, + now: trusted_server_core::ec::eid_types::UnixSeconds::new(1_700_000_000), + }; + + let request = refreshing_provider() + .build_request(&lookup) + .expect("should build the refresh request"); + + assert_eq!(request.method, http::Method::POST, "should post"); + assert_eq!( + request.url.as_str(), + REFRESH_ENDPOINT, + "should call the endpoint" + ); + assert_eq!( + request.body.as_ref(), + b"example-refresh-token", + "should post the refresh token only" + ); + assert_eq!( + request.context, + Some(BASE64.encode([7_u8; 32])), + "should keep the response key for decryption" + ); + } + + #[test] + fn makes_no_request_without_an_endpoint() { + let entry = stored_entry(); + let sources = [SourceDomain::try_from("uid2.example.com").expect("should build source")]; + let owner = ProviderId::new("uid2"); + let consent = ConsentContext::default(); + let lookup = LookupInput { + ec_id: "ec-example", + consent: &consent, + client: ClientSignals { + ip: None, + user_agent: None, + }, + publisher_domain: "www.example.com", + due_sources: &sources, + current: OwnedEntries::new(Some(&entry), &sources, &owner), + cookies: RequestCookies::default(), + hashed_email: None, + now: trusted_server_core::ec::eid_types::UnixSeconds::new(1_700_000_000), + }; + + assert!( + provider(&json!({})).build_request(&lookup).is_err(), + "should never generate a token" + ); + } + + fn encrypted_reply(reply: &serde_json::Value) -> String { + let cipher = Aes256Gcm::new_from_slice(&[7_u8; 32]).expect("should build cipher"); + let iv = [1_u8; IV_BYTES]; + let ciphertext = cipher + .encrypt( + &Nonce::try_from(&iv[..]).expect("should build nonce"), + reply.to_string().as_bytes(), + ) + .expect("should encrypt"); + BASE64.encode([&iv[..], &ciphertext].concat()) + } + + fn reply<'a>(body: &'a str, key: &'a str, headers: &'a HeaderMap) -> ProviderResponse<'a> { + ProviderResponse { + status: http::StatusCode::OK, + headers, + body: body.as_bytes(), + context: Some(key), + } + } + + #[test] + fn decrypts_a_refreshed_identity() { + let body = encrypted_reply(&json!({ + "status": "success", + "body": { + "advertising_token": "AgAAAArefreshedtoken", + "refresh_token": "next-refresh-token", + "identity_expires": 4_700_100_000_000_u64, + "refresh_from": 4_700_090_000_000_u64, + "refresh_expires": 4_700_200_000_000_u64, + "refresh_response_key": BASE64.encode([9_u8; 32]), + }, + })); + + let (key, headers) = (BASE64.encode([7_u8; 32]), HeaderMap::new()); + + let outcome = refreshing_provider() + .parse_response(reply(&body, &key, &headers)) + .expect("should parse the reply"); + + let LookupOutcome::Found(eids) = outcome else { + panic!("should find the refreshed token"); + }; + let uid = &eids[0].uids[0]; + assert_eq!(uid.id, "AgAAAArefreshedtoken", "should store the new token"); + assert!( + matches!(uid.expiry, ProviderExpiry::At(at) if at.get() == 4_700_100_000), + "should expire at the new identity_expires" + ); + let refresh = uid + .refresh + .as_ref() + .expect("should keep the next refresh state"); + let state: RefreshState = + serde_json::from_str(&refresh.state).expect("should parse the state"); + assert_eq!( + state.token, "next-refresh-token", + "should keep the next refresh token" + ); + assert_eq!( + state.key, + BASE64.encode([9_u8; 32]), + "should keep the next key" + ); + } + + #[test] + fn reports_an_opt_out_from_the_refresh() { + let body = encrypted_reply(&json!({ "status": "optout" })); + + let (key, headers) = (BASE64.encode([7_u8; 32]), HeaderMap::new()); + + let outcome = refreshing_provider() + .parse_response(reply(&body, &key, &headers)) + .expect("should parse the reply"); + + assert!( + matches!(outcome, LookupOutcome::OptedOut), + "should remove the stored token on opt-out" + ); + } + + #[test] + fn rejects_a_reply_under_another_key() { + let body = encrypted_reply(&json!({ "status": "optout" })); + + let (key, headers) = (BASE64.encode([8_u8; 32]), HeaderMap::new()); + + assert!( + refreshing_provider() + .parse_response(reply(&body, &key, &headers)) + .is_err(), + "should reject a reply that does not decrypt" + ); + } +} diff --git a/crates/trusted-server-identity-providers/src/unifiedid.rs b/crates/trusted-server-identity-providers/src/unifiedid.rs new file mode 100644 index 000000000..9e065fe43 --- /dev/null +++ b/crates/trusted-server-identity-providers/src/unifiedid.rs @@ -0,0 +1,224 @@ +//! Unified ID identity provider: reads The Trade Desk ID (TDID) from the +//! first-party cookie Prebid's `unifiedIdSystem` writes. +//! +//! Prebid stores a percent-encoded JSON object such as `{"TDID":"…"}` in +//! `pbjs-unifiedid` and sends the TDID as the `adserver.org` EID with agent +//! type 1, UID `ext` `{"rtiPartner":"TDID"}`, and `inserter`, `matcher` set +//! to `adserver.org` with `mm` 4. No server call is made. + +use std::borrow::Cow; + +use error_stack::{Report, ResultExt as _}; +use serde::Deserialize; +use serde_json::value::RawValue; +use trusted_server_core::ec::eid_types::SourceDomain; +use trusted_server_core::ec::provider::{ + CookieRead, EidLevelFields, IdentityProvider, LookupInput, LookupOutcome, ProviderCapabilities, + ProviderCommon, ProviderConfigError, ProviderError, ProviderKind, ProviderRequest, + ProviderResponse, RequestCookies, +}; + +use crate::common::{ + CookieReader, bounded_expiry, cookie_capabilities, device_eid, is_usable_id, percent_decoded, +}; + +/// `inserter` and `matcher` Prebid sets on the TDID EID. +const TDID_INSERTER: &str = "adserver.org"; + +/// `mm` (match method) Prebid sets on the TDID EID. +const TDID_MATCH_METHOD: i32 = 4; + +/// UID `ext` Prebid puts on the TDID. +const TDID_UID_EXT: &str = r#"{"rtiPartner":"TDID"}"#; + +/// The Unified ID provider kind. +pub struct UnifiedId; + +/// Settings for a `unifiedid` provider. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct UnifiedIdSettings { + /// Cookies holding the Unified ID object, read in order. + #[serde(default = "UnifiedIdSettings::default_cookie_names")] + pub cookie_names: Vec, +} + +impl UnifiedIdSettings { + fn default_cookie_names() -> Vec { + vec!["pbjs-unifiedid".to_owned()] + } +} + +/// A built Unified ID provider. +#[derive(Debug)] +pub struct UnifiedIdProvider { + capabilities: ProviderCapabilities, + source: SourceDomain, + cookies: CookieReader, + uid_ext: &'static RawValue, +} + +impl ProviderKind for UnifiedId { + const KIND: &'static str = "unifiedid"; + type Settings = UnifiedIdSettings; + type Provider = UnifiedIdProvider; + + fn build( + common: &ProviderCommon<'_>, + settings: UnifiedIdSettings, + ) -> Result> { + if settings.cookie_names.is_empty() { + return Err(Report::new(ProviderConfigError::Invalid) + .attach("a unifiedid provider needs at least one cookie name")); + } + let (mut capabilities, source) = cookie_capabilities(common, Self::KIND)?; + capabilities.shape.eid_level_fields = EidLevelFields::Preserved; + let uid_ext = serde_json::from_str::<&'static RawValue>(TDID_UID_EXT) + .change_context(ProviderConfigError::Invalid)?; + Ok(UnifiedIdProvider { + capabilities, + source, + cookies: CookieReader::own(common, settings.cookie_names), + uid_ext, + }) + } +} + +/// The field of the Unified ID cookie this provider reads. +#[derive(Deserialize)] +struct UnifiedIdCookie { + #[serde(rename = "TDID")] + tdid: String, +} + +/// Decodes one cookie value into the TDID. +fn tdid(value: &str) -> Option { + let cookie: UnifiedIdCookie = serde_json::from_str(&percent_decoded(value)).ok()?; + is_usable_id(&cookie.tdid).then_some(cookie.tdid) +} + +impl IdentityProvider for UnifiedIdProvider { + fn capabilities(&self) -> &ProviderCapabilities { + &self.capabilities + } + + fn build_request( + &self, + _lookup: &LookupInput<'_>, + ) -> Result> { + Err(Report::new(ProviderError::Request).attach("unifiedid makes no lookup")) + } + + fn parse_response<'body>( + &self, + _response: ProviderResponse<'body>, + ) -> Result, Report> { + Err(Report::new(ProviderError::InvalidResponse).attach("unifiedid makes no lookup")) + } + + fn read_request_cookies<'cookie>( + &self, + cookies: RequestCookies<'cookie>, + ) -> CookieRead<'cookie> { + let Some(id) = self.cookies.read(cookies, tdid) else { + return CookieRead::Absent; + }; + let mut eid = device_eid(&self.source, Cow::Owned(id), bounded_expiry(None)); + eid.inserter = Some(Cow::Borrowed(TDID_INSERTER)); + eid.matcher = Some(Cow::Borrowed(TDID_INSERTER)); + eid.mm = Some(TDID_MATCH_METHOD); + for uid in &mut eid.uids { + uid.ext = Some(Cow::Borrowed(self.uid_ext)); + } + CookieRead::Found(vec![eid]) + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + use trusted_server_core::ec::provider::build_kind; + use trusted_server_core::settings::{IdentityProviderConfig, ProviderSecrets}; + + use super::*; + + fn provider() -> Box { + let config: IdentityProviderConfig = serde_json::from_value(json!({ + "id": "unifiedid", + "kind": "unifiedid", + "sources": ["unifiedid.example.com"], + })) + .expect("should parse provider config"); + build_kind::(&config, &ProviderSecrets::default()) + .expect("should build provider") + } + + fn header(cookie: &serde_json::Value) -> String { + format!( + "pbjs-unifiedid={}", + urlencoding::encode(&cookie.to_string()) + ) + } + + #[test] + fn reads_the_tdid_with_prebid_fields() { + let cookie = json!({ + "TDID": "tdid-example", + "TDID_LOOKUP": "TRUE", + "TDID_CREATED_AT": "2026-01-01T00:00:00", + }); + let header = header(&cookie); + + let eids = provider() + .read_request_cookies(RequestCookies::new(&header)) + .into_found() + .expect("should read the TDID"); + + let eid = &eids[0]; + assert_eq!(eid.source, "unifiedid.example.com", "should use the source"); + assert_eq!( + eid.inserter.as_deref(), + Some("adserver.org"), + "should set inserter" + ); + assert_eq!( + eid.matcher.as_deref(), + Some("adserver.org"), + "should set matcher" + ); + assert_eq!(eid.mm, Some(4), "should set mm"); + let uid = &eid.uids[0]; + assert_eq!(uid.id, "tdid-example", "should store only the TDID"); + assert_eq!(uid.atype, Some(1), "should be device-based"); + assert_eq!( + uid.ext.as_deref().map(RawValue::get), + Some(r#"{"rtiPartner":"TDID"}"#), + "should mark the rtiPartner" + ); + } + + #[test] + fn ignores_missing_malformed_and_blank_ids() { + for cookie in [ + json!({}), + json!({ "TDID": "" }), + json!({ "TDID": "0" }), + json!({ "TDID": 5 }), + ] { + assert!( + provider() + .read_request_cookies(RequestCookies::new(&header(&cookie))) + .into_found() + .is_none(), + "should ignore {cookie}" + ); + } + assert!( + provider() + .read_request_cookies(RequestCookies::new("pbjs-unifiedid={broken")) + .into_found() + .is_none(), + "should ignore malformed JSON" + ); + } +} diff --git a/docs/guide/api-reference.md b/docs/guide/api-reference.md index e6103be39..840c58cda 100644 --- a/docs/guide/api-reference.md +++ b/docs/guide/api-reference.md @@ -876,6 +876,7 @@ This lookup is implemented only by the Fastly adapter because the identity graph - `ec_id`, `store`, and `generation` identify the raw KV lookup. - `entry` preserves the stored JSON shape, including unknown and legacy fields. Derived `created_iso` and `consent.updated_iso` fields are added only when absent. - `metadata` preserves the stored metadata JSON shape. +- Provider refresh state is private: every `refresh.state` in `entry` is shown as `"[redacted]"`, while `refresh.from` stays visible. - `tombstone` reports whether consent has been withdrawn. It is absent when the entry body cannot be parsed as JSON or deserialized as the typed EC schema. - `auction.eids` previews the partner EIDs the stored record can contribute; `auction.skipped` explains filtered IDs. - `entry_error`, `metadata_error`, and `raw_body` keep malformed or schema-incompatible records inspectable. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index dd9912ec2..6f2171e2f 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -794,8 +794,8 @@ TRUSTED_SERVER__EC__EC_STORE=ec_identity_store ### Identity providers -`[[ec.identity_providers]]` entries look up a visitor's IDs from a provider's -server on publisher page navigations. Lookups start before the origin fetch, +`[[ec.identity_providers]]` entries collect a visitor's IDs from the vendor's +first-party cookies or a provider's server on publisher page navigations. Lookups start before the origin fetch, share the `identity_budget_ms` budget, and are collected before the response is finalized. Results go into the navigation's single EC row write. A lookup that times out or fails leaves the row unchanged. @@ -826,23 +826,32 @@ them, and batch sync rejects them and pull sync skips them. Browser-resolved IDs (`ts-eids`, `sharedId`) are never stored for an owned source, so a source a provider cleared on an opt-out is not refilled from the browser. -When a provider reports an opt-out, its sources are cleared and the row -remembers the opt-out for 30 days. The provider is not called for that EC -again until the 30 days pass. - -| Field | Type | Required | Description | -| --------------------- | ------- | -------- | ----------------------------------------------------------------------- | -| `id` | String | Yes | Stable provider ID used in logs, metrics and KV ownership | -| `kind` | String | Yes | Provider implementation; must be one this build includes | -| `enabled` | Boolean | No | Whether the provider runs (default `true`) | -| `endpoint` | String | Yes | HTTPS lookup URL; its host must be in `allowed_domains` | -| `allowed_domains` | Array | Yes | Hosts the provider may call | -| `sources` | Array | Yes | Source domains the provider owns; each needs an `[[ec.partners]]` entry | -| `tcf_vendor_id` | Integer | No | TCF vendor ID that must have vendor consent | -| `collect` | String | No | `"page"` (default) or `"after_response"`; see below | -| `rate_limit_per_hour` | Integer | No | Most lookups per EC hash per hour; unset means unlimited. See below | -| `secrets.token` | String | No | Key in `trusted_server_secrets` for the provider's API token | -| `settings` | Table | No | Provider-specific settings, parsed by the provider kind | +When a provider reports an opt-out, its sources are cleared. An opt-out the +vendor's server reports is also remembered in the row for 30 days, and the +provider is not called for that EC again until the 30 days pass, unless the +vendor's cookies hold an ID again: that means the visitor opted back in, so the +opt-out is forgotten. An opt-out read from the vendor's cookies is not +remembered, because the cookies are read again on every navigation. + +Some kinds keep private refresh state with a UID, such as a vendor refresh +token, and refresh it server side once it is due, while the UID is still live. +Refresh state is never served, mirrored, forwarded or shown: the admin view +shows it as `[redacted]`. Kinds without a vendor server may generate an ID into +the EC row when the cookies hold none; no cookie is set. + +| Field | Type | Required | Description | +| --------------------- | ------- | -------- | ------------------------------------------------------------------------------------- | +| `id` | String | Yes | Stable provider ID used in logs, metrics and KV ownership | +| `kind` | String | Yes | Provider implementation; must be one this build includes | +| `enabled` | Boolean | No | Whether the provider runs (default `true`) | +| `endpoint` | String | No | HTTPS lookup URL, for kinds that make a lookup; its host must be in `allowed_domains` | +| `allowed_domains` | Array | No | Hosts the provider may call; required with an `endpoint` | +| `sources` | Array | Yes | Source domains the provider owns; each needs an `[[ec.partners]]` entry | +| `tcf_vendor_id` | Integer | No | TCF vendor ID that must have vendor consent | +| `collect` | String | No | `"page"` (default) or `"after_response"`; see below | +| `rate_limit_per_hour` | Integer | No | Most lookups per EC hash per hour; unset means unlimited. See below | +| `secrets.token` | String | No | Key in `trusted_server_secrets` for the provider's API token | +| `settings` | Table | No | Provider-specific settings, parsed by the provider kind | **Example**: @@ -905,6 +914,13 @@ build, and an older writer drops a row's `eids` and provider opt-outs when it rewrites the row. ::: +`ts config validate`, `ts config push` and `ts config diff` also build every +enabled provider of a registered kind, so a provider-specific setting error +(for example a `liveramp` `endpoint` without `placement_id` and +`custom_id_namespace`) fails the command before deploy. At runtime the same +error would disable every identity provider for each request, with an error +log. + ### Field Details #### `passphrase` diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-connectid.md b/docs/superpowers/plans/2026-11-02-identity-provider-connectid.md new file mode 100644 index 000000000..2d70da7dd --- /dev/null +++ b/docs/superpowers/plans/2026-11-02-identity-provider-connectid.md @@ -0,0 +1,36 @@ +# Yahoo ConnectID Identity Provider Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Status:** Implemented; checkboxes are not updated. + +**Goal:** Store Yahoo ConnectID from Yahoo's own first-party cookie, as an `[[ec.identity_providers]]` entry with `kind = "connectid"`. No login and no server call. + +**Architecture:** `crates/trusted-server-identity-providers/src/connectid.rs` holds `ConnectId` (a `ProviderKind`), `ConnectIdSettings` and `ConnectIdProvider`. The provider implements only `IdentityProvider::read_request_cookies`, using the cookie helpers in `common.rs`; `build_request` always fails, so no lookup is sent. + +**Spec:** `docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md`, section 7. + +## Cookie + +- Default cookie `connectId`; `cookie_names` overrides the list, read in order. +- The value is JSON, optionally percent-encoded: `connectId`, `he`, `puid`, `lastSynced`, `lastUsed`, `ttl` (timestamps and TTL in milliseconds). +- Only `connectId`, `lastSynced` and `ttl` are read. `he` and `puid` are never deserialized, so they are never stored. +- A missing `connectId` or `lastSynced`, a non-string ID, a blank or opt-out ID, or malformed JSON reads as no ID. + +No Lockr capability: Lockr AIM is not documented to deliver ConnectID, so a `[settings.lockr]` table is rejected. + +## Stored ID + +- One EID for the single configured source, one UID with `atype = 3`. +- Expiry `ProviderExpiry::At(lastSynced + min(ttl, 14 days))`; as in Prebid, a missing or zero `ttl` means 14 days, a missing `lastSynced` is stored as `ProviderExpiry::After(14 days)`, and a lowercase `connectid` key is accepted. The provider checks expiry with the clock so an expired cookie does not hide a later live one; the coordinator's validation also drops the ID once that time has passed. + +## Not ported from the reference provider + +- The reference calls Yahoo's sync endpoint, which needs only `pixelId` (`he` is optional), but its useful matches come from the hashed email. The logged-in experience is out of scope, so nothing is ported; this provider reads the `connectId` cookie only. + +## Tasks + +- [ ] Add `connectid.rs` and register the kind in `is_registered_kind` and `build`. +- [ ] Unit tests: ID and expiry from `lastSynced + ttl`, `he`/`puid` not stored, TTL capped and defaulted, missing, malformed and opt-out cookies ignored. +- [ ] Add a commented `kind = "connectid"` example to `trusted-server.example.toml`. +- [ ] Run `cargo test-fastly` and `cargo clippy-fastly`. diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-criteo.md b/docs/superpowers/plans/2026-11-02-identity-provider-criteo.md new file mode 100644 index 000000000..4bc43121a --- /dev/null +++ b/docs/superpowers/plans/2026-11-02-identity-provider-criteo.md @@ -0,0 +1,35 @@ +# Criteo Identity Provider Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Status:** Implemented; checkboxes are not updated. + +**Goal:** Store the Criteo ID from Criteo's first-party cookie, as an `[[ec.identity_providers]]` entry with `kind = "criteo"`. No server call. + +**Architecture:** `crates/trusted-server-identity-providers/src/criteo.rs` holds `Criteo` (a `ProviderKind`), `CriteoSettings` and `CriteoProvider`. The provider implements only `IdentityProvider::read_request_cookies`; `build_request` always fails. + +**Spec:** `docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md`, section 7. + +## Cookie + +- Prebid's `criteoIdSystem` keeps Criteo's bid ID in the `cto_bidid` cookie (and local storage) and sends the `decodeURIComponent`-decoded value as the `criteo.com` EID (unwrapping a `{"criteoId":…}` value). `cto_bundle` is Criteo's own sync state and is not an ID. +- Default `cookie_names = ["cto_bidid"]`, read in order. The value is percent-decoded and stored as Prebid sends it. +- A blank or opt-out value reads as no ID. Criteo does not set `cto_bidid` on every page view, so the ID arrives on a later request. + +No Lockr capability. + +## Stored ID + +- One EID for the single configured source, one UID with `atype = 1`. +- Default bounded expiry; a cookie repeating the held ID is not rewritten, so the expiry renews only when the source is due. + +## Not ported + +- Criteo has no public publisher S2S ID API; Criteo's `gum.criteo.com` sync call needs Criteo's third-party cookie, so it is not made; the first-party cookie is enough. + +## Tasks + +- [ ] Add `criteo.rs`; register the kind in `is_registered_kind` and `build`. +- [ ] Unit tests: value stored decoded, `criteoId` object unwrapped, missing and blank cookies ignored. +- [ ] Add a commented `kind = "criteo"` example to `trusted-server.example.toml`. +- [ ] Run `cargo test-fastly` and `cargo clippy-fastly`. diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-id5.md b/docs/superpowers/plans/2026-11-02-identity-provider-id5.md new file mode 100644 index 000000000..a4d741ccb --- /dev/null +++ b/docs/superpowers/plans/2026-11-02-identity-provider-id5.md @@ -0,0 +1,50 @@ +# ID5 Identity Provider Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Status:** Implemented; checkboxes are not updated. + +**Goal:** Resolve the ID5 ID at the edge as an anonymous `[[ec.identity_providers]]` entry with `kind = "id5"`, storing the complete EIDs ID5 returns. + +**Architecture:** `crates/trusted-server-identity-providers/src/id5.rs` holds `Id5` (a `ProviderKind`), `Id5Settings` and `Id5Provider` (an `IdentityProvider`), plus one `build` arm in `lib.rs`. The provider is pure: `build_request` builds one JSON `POST`, and `parse_response` borrows EIDs from the body. The coordinator from PR A (`ec::coordinator`) applies consent, the source gate, validation, expiry and persistence. + +**Spec:** `docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md`, section 7. + +**Changes from the earlier ID5 plan (built on the old descriptor framework):** + +- No hashed email and no `pd` field: every source is `SourceInput::Anonymous`. +- A reply's non-empty `signature` is kept as private refresh state, due a tenth of the TTL before the UIDs expire, and sent back as `s`, as Prebid's ID5 module does. +- Endpoint, sources and the TCF vendor ID come from the common provider config, not hard-coded defaults. +- The legacy `universal_uid` fallback goes to the first configured source. An opted-out `universal_uid` of `0` is reported as `LookupOutcome::OptedOut`, detected on the reply as sent (not percent-decoded). + +## Request + +- `POST /.json`, with `Content-Type: application/json` and `Origin: https://`. +- Body, following ID5's server-side client (its Prebid Server module): `partner`, `provider` (`provider_name`, default the publisher domain), `domain` (the publisher domain), `origin = "trusted-server"`, `gdpr`, and when present `gdpr_consent`, `us_privacy`, `gpp_string`, `gpp_sid` (comma-joined), `ua`, and `ip` (IPv4) or `ipv6` (only when routable, see `common::is_routable_ip`). +- Confirm the `ip` and `ua` field names with ID5 for each partner account. + +## Response + +- A non-2xx status or an unparseable body is `ProviderError::InvalidResponse`. +- `ids..eid` entries whose source is configured become `ProviderEid`s; every field, including `ext`, is borrowed from the body. +- `cache_control.max_age_sec` becomes `ProviderExpiry::After` for every UID. +- With no EID for the first source, a non-empty, non-`0` `universal_uid` becomes an EID with `atype = 1` and, when the reply has `ext.linkType`, UID `ext` `{"linkType": ...}`, as in the reference provider and Prebid. + +## First-party cookie + +ID5's source is provider-owned, so the browser `ts-eids` copy is dropped. Prebid's ID5 module stores the ID in a first-party cookie instead, which `read_request_cookies` reads before any lookup. + +- Setting `cookie_names` (default `["id5id"]`), tried in order. +- `id5` declares the Lockr capability (see `2026-11-02-identity-provider-lockr-capability.md`): with `[settings.lockr] enabled = true`, the Lockr cookies (default `["id5id"]`) are read after `cookie_names` and before the lookup, with the same parser. +- A value that is JSON after percent-decoding is parsed with the same code as a lookup response (lenient entries, opt-out `0` dropped, normalized sources, `cache_control.max_age_sec` bounded). When the value was percent-encoded, entries, `ext` objects included, are copied out of the decoded text. +- Any other value is a bare ID: one UID on the first source with `atype = 1` and the default TTL. +- `signature` is never stored as a UID, only as refresh state. A cookie that yields no usable UID is ignored, and the lookup runs as usual; a kept UID skips the lookup. + +## Tasks + +- [ ] Add `common.rs` with `is_routable_ip`, `join_section_ids` and the borrowed `VendorEid`. +- [ ] Add `id5.rs` and the `build` arm. +- [ ] Unit tests: request shape, private IP dropped, complete EIDs parsed and unconfigured sources dropped, legacy fallback, opted-out and empty replies are no match, error status and malformed body rejected. +- [ ] Read `cookie_names` in `read_request_cookies`; tests for plain and percent-encoded reply JSON, a bare ID from a configured name, and opted-out or malformed cookies reading as no ID. +- [ ] Add a commented `kind = "id5"` example, with `cookie_names`, to `trusted-server.example.toml`. +- [ ] Run `cargo test-fastly` and `cargo clippy-fastly`. diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md b/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md new file mode 100644 index 000000000..d220dd019 --- /dev/null +++ b/docs/superpowers/plans/2026-11-02-identity-provider-liveintent.md @@ -0,0 +1,47 @@ +# LiveIntent Identity Provider Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Status:** Implemented; checkboxes are not updated. + +**Goal:** Resolve `LiveIntent`'s IDs at the edge as a no-login `[[ec.identity_providers]]` entry with `kind = "liveintent"`, storing one EID per resolved attribute. + +**Architecture:** `crates/trusted-server-identity-providers/src/liveintent.rs` holds `LiveIntent` (a `ProviderKind`), `LiveIntentSettings` and `LiveIntentProvider` (an `IdentityProvider`), plus one `build` arm in `lib.rs`. The provider is pure: `build_request` builds one `GET` to the identity exchange, and `parse_response` maps returned attributes to sources. The coordinator from PR A applies consent, the source gate, validation, expiry and persistence. + +**Spec:** `docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md`, section 7. + +**Changes from the earlier LiveIntent plan (built on the old descriptor framework):** + +- No login: no hashed email. The first-party `LiveIntent` cookie (`_lc2_fpi` by default) is sent as `duid`, read from `LookupInput::cookies`. +- The attribute-to-source table is fixed, as in the reference provider; `settings.resolve` picks attributes from it, and every picked attribute's source must be one of the provider's `sources`. +- No page URL in `LookupInput`, so `pu` is the publisher origin. +- EID-level fields are preserved. As in Prebid, every attribute other than `nonId` is a partner ID whose UID carries `ext` `{"provider":"liveintent.com"}`; the Trade Desk ID's UID also carries `rtiPartner: "TDID"` and its EID `matcher = "liveintent.com"`. The reply's `segments` become the `nonId` EID's `ext` (`{"segments": ...}`), which needs the PR A change letting a provider return an owned EID `ext`. +- The `Expires` header becomes `ProviderExpiry::After(Expires - Date)`; the provider never reads the clock. + +## Settings + +| Key | Default | Meaning | +| ---------------- | -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `publisher_id` | `any` | Path segment after the distributor. | +| `distributor_id` | none | A `did-...` ID; sent as `did` and used as the path. LiveIntent rejects any other form, so build rejects it. The path uses `prebid` when unset. | +| `resolve` | `["nonId"]` | Attributes to ask for, from the fixed table: `nonId` liveintent.com, `bidswitch`, `medianet`, `magnite`, `index`, `openx`, `pubmatic`, `sovrn`, `sharethrough`, `sonobi`, `triplelift`, `zetassp`, `vidazoo`, `nexxen`, `uid2`, `thetradedesk` adserver.org. | +| `cookie_names` | `["_lc2_fpi"]` | First-party cookies; the first present one is sent as `duid`. | + +No Lockr capability: Lockr AIM is not documented to deliver LiveIntent IDs, and the lookup uses LiveIntent's own signals, so a `[settings.lockr]` table is rejected. + +## Request + +- `GET //` with `Origin: https://`. +- Query: `did`, `duid`, `gdpr`, `gdpr_consent`, `us_privacy`, `gpp_s`, `gpp_as`, `pu`, `pip` or `pip6` (base64url of a routable client IP), and one `resolve` per attribute whose source is due. + +## Response + +- `204 No Content` is `LookupOutcome::NoMatch`. LiveIntent answers `204` to anonymous `gdpr=1` lookups. On the `prebid` path it returns only `nonId`; with the publisher's own `distributor_id` (for example `did-0123`) and the `_lc2_fpi` first-party ID as `duid`, it also returns partner IDs without its third-party cookie, so a publisher wanting partner IDs must set `distributor_id`; another non-2xx status or a non-object body is `ProviderError::InvalidResponse`. +- Each resolved attribute with a non-empty string value becomes one EID with `atype = 3` (the resolved TDID gets `atype = 1`, as in Prebid), borrowed from the body; `segments`, the TDID `matcher` and the UID `ext` values are added as above. Segments whose wrapped `ext` exceeds `MAX_EXT_BYTES` are left out so the `nonId` EID is kept. + +## Tasks + +- [ ] Add `liveintent.rs` and the `build` arm. +- [ ] Unit tests: unknown attribute and unconfigured source rejected, `duid` from the first present cookie, segments/matcher/ext mapping, request shape with only due attributes, attributes mapped with the `Expires` expiry, no-content and empty IDs are no match, error status and malformed body rejected. +- [ ] Add a commented `kind = "liveintent"` example to `trusted-server.example.toml`. +- [ ] Run `cargo test-fastly` and `cargo clippy-fastly`. diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md b/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md new file mode 100644 index 000000000..8708d5c8e --- /dev/null +++ b/docs/superpowers/plans/2026-11-02-identity-provider-liveramp.md @@ -0,0 +1,42 @@ +# LiveRamp Identity Provider Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Status:** Implemented; checkboxes are not updated. + +**Goal:** Store the LiveRamp RampID envelope the browser already sends in LiveRamp's own first-party cookie, as an `[[ec.identity_providers]]` entry with `kind = "liveramp"`. No login. With an `endpoint`, a visitor without an envelope cookie is also looked up server to server. + +**Architecture:** `crates/trusted-server-identity-providers/src/liveramp.rs` holds `LiveRamp` (a `ProviderKind`), `LiveRampSettings` and `LiveRampProvider`. The provider reads the envelope cookies in `IdentityProvider::read_request_cookies`; without an `endpoint`, `build_request` fails, so no lookup is sent. The coordinator reads cookies on every navigation (a cookie repeating the held UID is not rewritten) and applies consent, validation, expiry and persistence exactly as for lookup results. + +**Spec:** `docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md`, section 7. + +## Cookie + +- Default cookies `_lr_env` then `idl_env`; `cookie_names` overrides the list, read in order. +- `_lr_env` (written by ATS) is base64 or base64url (padded or not, optionally percent-encoded) of a JSON object. Only its string `envelope` field is used, unchanged. +- `idl_env` is where Prebid's `identityLink` ID module stores the envelope it fetched, bare. A value in the base64 alphabet that does not decode to JSON is taken as such a bare envelope, unchanged. +- A blank or opt-out envelope (`0`, all zeros), a JSON body without a string `envelope`, and any value outside the base64 alphabet read as no ID. + +## Stored ID + +- One EID for the single configured source, one UID with `atype = 3`. +- Expiry `ProviderExpiry::After(15 days)` through `bounded_expiry`. +- Optional server lookup (with `endpoint`): `GET /api/identity/v2/envelope` with `pid` (`placement_id`), `it=15`, `iv=:`, `atype=3` (server to server; `2` is the ATS API), consent as `ct`/`cv` (TCF `4`, else US Privacy `3`), `gpp` and `gpp_sid` (the first section ID only); headers `Origin` (publisher) and `X-Forwarded-For` (routable visitor IP, else no lookup). `200 {"envelopes":[{"type":19,"value":…}]}`: the `RampID` entry (`type` 19) is stored like the cookie, and a reply without one is no match; `204` is an opt-out; a blank envelope is no match; other statuses fail. Without `endpoint`, no server call. LiveRamp must enable Custom ID for the placement and approve the origin. LiveRamp resolves a Custom ID only through a mapping file the publisher sends it, which ties each Custom ID to a hashed email; without that file, or for an EC ID absent from it, the lookup finds nothing. + +## Lockr + +`liveramp` declares the Lockr capability (see `2026-11-02-identity-provider-lockr-capability.md`). With `[settings.lockr] enabled = true`, the Lockr cookies (default `["_lr_env"]`) are read after `cookie_names` with the same envelope parser. `cookie_names` may be empty only when Lockr is enabled. + +## Not ported from the reference provider + +- Hashed-email and hashed-phone lookups (`it=4`, `it=11`): out of scope (no logged-in experience). + +## Tasks + +- [x] Custom ID envelope lookup with `endpoint`, `placement_id` and `custom_id_namespace`, with tests. + +- [ ] Add `cookie_capabilities`, `cookie_eid`, `is_usable_id` and `percent_decoded` to `common.rs`. +- [ ] Add `liveramp.rs` and register the kind in `is_registered_kind` and `build`. +- [ ] Unit tests: base64 and base64url envelopes, configured cookie name, missing, malformed and opt-out cookies ignored, one-source and cookie-name config checks, no lookup built. +- [ ] Add a commented `kind = "liveramp"` example to `trusted-server.example.toml`. +- [ ] Run `cargo test-fastly` and `cargo clippy-fastly`. diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-lockr-capability.md b/docs/superpowers/plans/2026-11-02-identity-provider-lockr-capability.md new file mode 100644 index 000000000..85627a244 --- /dev/null +++ b/docs/superpowers/plans/2026-11-02-identity-provider-lockr-capability.md @@ -0,0 +1,34 @@ +# Lockr Cookie Capability Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Status:** Implemented; checkboxes are not updated. + +**Goal:** Read vendor IDs that Lockr AIM writes into first-party cookies on the publisher domain, as an explicit, reusable capability of the provider kinds Lockr delivers (ID5, LiveRamp, UID2), instead of ad-hoc cookie names per provider. + +**Architecture:** Everything lives in `crates/trusted-server-identity-providers/src/common.rs`; core only passes the configured provider ID to builds (`ProviderCommon::id`) for logging. + +- `LockrSettings` (`enabled`, default `false`; `cookie_names`, default the vendor's known Lockr key), parsed from `[ec.identity_providers.settings.lockr]` as a field of each capable kind's settings. +- `ReadsLockr`: a trait a `ProviderKind` implements to declare the capability, with `LOCKR_COOKIE_NAMES` (`id5id` for ID5, `_lr_env` for LiveRamp, `__uid2_advertising_token` for UID2). `reads_lockr(kind)` in `lib.rs` lists the same kinds. +- `CookieReader`: holds the own cookie names and, when Lockr is enabled, the Lockr names not already among them. `read` hands each value to the provider's existing parser: own cookies first, then Lockr cookies. Lookup kinds look up only when neither yields an ID. +- A read from a Lockr cookie logs at debug with the provider ID, the cookie name and origin `lockr`. +- Lockr cookies are not read where GDPR applies (`RequestCookies::gdpr_applies`, set by the coordinator from the request consent): Prebid's `lockrAIMIdSystem` stops when `gdprApplies` is true, so a Lockr cookie on such a request is a leftover, not a current delivery. Own cookies are still read. +- Each default Lockr key equals one of the kind's own default cookie names, which are already read, so enabling the capability with the defaults reads nothing extra. Operators list the key configured in Lockr when it differs. + +**Lockr storage format (from Prebid's `lockrAIMIdSystem`):** each vendor token is stored as the raw `advertising_token` under the server-configured `key_name`, in a cookie and or localStorage. Lockr's own `ilui`, `_expiry` and `lockr_expiry_keys` keys are localStorage only. No `inserter` field and no opt-out marker are stored. + +**Gating:** kinds without the capability (ConnectID, LiveIntent) have `deny_unknown_fields` settings without a `lockr` field, so a `lockr` table fails the provider build at startup. Lockr enabled with an empty `cookie_names` is rejected. + +**Own-cookie settings:** every cookie-reading kind takes a `cookie_names` list (UID2's `token_cookie_name` and ConnectID's `cookie_name` are renamed). Lockr names move out of them into `[settings.lockr]`. + +## Not ported from the reference provider + +- The reference captures proxied Lockr API responses: a `revoke-consent` call opts the visitor out of Lockr's sources, and token replies are mapped to sources with Lockr provenance (`inserter`, `mm`) on ID5. The framework has no response-capture hook or opt-out state, and Lockr's default ID5 key is the same `id5id` cookie Prebid writes, so a Lockr-read ID cannot be told apart for provenance. Neither is ported: Lockr stores neither in browser storage, so a server reading cookies cannot see them. + +## Tasks + +- [ ] Add `ProviderCommon::id` in core and set it in `build_kind`. +- [ ] Rename UID2 `token_cookie_name` and ConnectID `cookie_name` to `cookie_names`. +- [ ] Add `LockrSettings`, `ReadsLockr` and `CookieReader` to `common.rs`; wire ID5, LiveRamp and UID2; use `CookieReader::own` for ConnectID. +- [ ] Tests: capability gating per kind, empty Lockr list rejected, Lockr disabled ignores the cookie, a custom Lockr name is read, own cookie before Lockr, no cookie leaves ID5 to the lookup. +- [ ] Update `trusted-server.example.toml` and the provider plans. diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-sharedid.md b/docs/superpowers/plans/2026-11-02-identity-provider-sharedid.md new file mode 100644 index 000000000..cd5544090 --- /dev/null +++ b/docs/superpowers/plans/2026-11-02-identity-provider-sharedid.md @@ -0,0 +1,34 @@ +# Prebid SharedID Identity Provider Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Status:** Implemented; checkboxes are not updated. + +**Goal:** Store the Prebid SharedID (publisher common ID) from its first-party cookie, as an `[[ec.identity_providers]]` entry with `kind = "sharedid"`. No server call. When no cookie ID and no live stored ID exist, TS generates a random UUID v4 into KV only (no cookie set); a later Prebid cookie replaces it. + +**Architecture:** `crates/trusted-server-identity-providers/src/sharedid.rs` holds `SharedId` (a `ProviderKind`), `SharedIdSettings` and `SharedIdProvider`. The provider implements `IdentityProvider::read_request_cookies` and `generate`; `build_request` always fails, so no lookup is sent. `common::device_eid` builds the agent type 1 EID. + +**Spec:** `docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md`, section 7. + +## Cookie + +- Prebid's `sharedIdSystem` writes a bare UUID to the publisher-configured cookie, usually `_sharedid`; older pages use `_pubcid`. +- Default `cookie_names = ["_sharedid", "_pubcid"]`, read in order. +- The `_pubcid_optout` cookie suppresses the ID, as in Prebid. +- A blank or opt-out (all-zero) value reads as no ID. + +No Lockr capability. + +## Stored ID + +- One EID for the single configured source, one UID with `atype = 1`, as Prebid sends `pubcid.org`. +- Cookie IDs: default bounded expiry; a cookie repeating the held ID is not rewritten, so the expiry renews only when the source is due. +- Generated IDs expire after 30 days (`MAX_TTL`), not the default 24 hours. When one expires while the row still holds it, it is renewed with the same ID rather than replaced by a new UUID, so it stays stable for as long as the row lives. + +## Tasks + +- [ ] Add `sharedid.rs` and `common::device_eid`; register the kind in `is_registered_kind` and `build`. +- [ ] Unit tests: `_sharedid` before `_pubcid`, opt-out cookie honoured, blank and opt-out IDs ignored. +- [ ] `generate` (UUID v4, 30-day expiry) + test. +- [ ] Add a commented `kind = "sharedid"` example to `trusted-server.example.toml`. +- [ ] Run `cargo test-fastly` and `cargo clippy-fastly`. diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md b/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md new file mode 100644 index 000000000..d21a690c2 --- /dev/null +++ b/docs/superpowers/plans/2026-11-02-identity-provider-uid2.md @@ -0,0 +1,48 @@ +# UID2 Identity Provider Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Status:** Implemented; checkboxes are not updated. + +**Goal:** Store the UID2 advertising token from first-party cookies, as an `[[ec.identity_providers]]` entry with `kind = "uid2"`, and, when `endpoint` is set, refresh it server side. No login; tokens are never generated. + +**Architecture:** `crates/trusted-server-identity-providers/src/uid2.rs` holds `Uid2` (a `ProviderKind`), `Uid2Settings` and `Uid2Provider`. `read_request_cookies` reads the token, using the cookie helpers in `common.rs`; with an `endpoint`, `build_request` posts the stored refresh token to `POST /v2/token/refresh` and `parse_response` decrypts the AES-256-GCM reply (12-byte IV first) using the response key passed in `ProviderRequest::context`. Without an `endpoint`, `build_request` fails. + +**Spec:** `docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md`, section 7. + +## Cookies + +- `identity_cookie_name` (default `__uid_2`): percent-encoded JSON from the UID2 SDK. `advertising_token` and `identity_expires` (milliseconds) are read. With an `endpoint`, `refresh_token`, `refresh_response_key` and `refresh_from` are kept as private refresh state (never served, mirrored or forwarded; the admin view shows it as `[redacted]`); without it they are not stored. `refresh_expires` is not read. +- `cookie_names` (default `["__uid2_advertising_token"]`): cookies holding a raw token, such as Prebid's, read in order only when the identity cookie yields nothing. +- A missing token or expiry, a blank or opt-out token, or malformed JSON reads as no ID. + +## Stored ID + +- One EID for the single configured source, one UID with `atype = 3`. +- From the identity cookie: `ProviderExpiry::At(identity_expires)`; the coordinator's validation drops an expired token. +- From the raw token cookie: `bounded_expiry(None)`, the shared default TTL. +- Refresh state `from = refresh_from`; the source becomes due then, and is refreshed once due, also while the cookie repeats the stored token (a cookie repeating the held token is not rewritten, so a failed refresh keeps the held refresh state). A refresh is due only while the token is still live, so a failing refresh stops at its expiry; `rate_limit_per_hour` also bounds it. An `optout` reply removes the stored token. + +## Lockr + +`uid2` declares the Lockr capability (see `2026-11-02-identity-provider-lockr-capability.md`). With `[settings.lockr] enabled = true`, the Lockr cookies (default `["__uid2_advertising_token"]`) are read last, after the identity cookie and `cookie_names`, with the raw-token parser. Lockr names already in `cookie_names` are not read twice. + +## Not ported from the reference provider + +- Token generation (needs a hashed email; out of scope). + +Risk: a server refresh does not rewrite `__uid_2`. Refreshing does not invalidate the previous token or refresh token, so the server refresh and the browser SDK's own refresh both succeed and fork into two token chains. Mitigations: + +- The provider sets `keeps_fresher_stored_uids`: a cookie token whose `identity_expires` is earlier than the live stored token's is ignored, so an older cookie never rolls a refreshed token back, and the refreshed token's own `refresh_from` decides the next server refresh. +- Once the SDK refreshes, its cookie token outlives the stored one and replaces it with its refresh state, so the row follows the SDK's chain again; the two chains do not alternate on every navigation. +- A failed refresh keeps the held token and refresh state, and due state is kept even past `refresh_from`; `rate_limit_per_hour` bounds the retries. + +Remaining risk: while the SDK has not refreshed, the server refresh keeps a second chain alive, one refresh per `refresh_from` interval. Operators running the SDK with `useCookie: true` can leave `endpoint` unset to rely on the SDK alone. + +## Tasks + +- [ ] Add `uid2.rs` and register the kind in `is_registered_kind` and `build`. +- [ ] Unit tests: token and expiry from `__uid_2` with refresh fields not stored, raw token fallback, configured cookie names, missing, malformed and opt-out cookies ignored. +- [ ] Refresh: `endpoint`, AES-256-GCM decrypt with the response key from `ProviderRequest::context`, `optout` reply as `LookupOutcome::OptedOut`, and `refresh_state` tests. +- [ ] Add a commented `kind = "uid2"` example to `trusted-server.example.toml`. +- [ ] Run `cargo test-fastly` and `cargo clippy-fastly`. diff --git a/docs/superpowers/plans/2026-11-02-identity-provider-unifiedid.md b/docs/superpowers/plans/2026-11-02-identity-provider-unifiedid.md new file mode 100644 index 000000000..927ee3867 --- /dev/null +++ b/docs/superpowers/plans/2026-11-02-identity-provider-unifiedid.md @@ -0,0 +1,36 @@ +# Unified ID Identity Provider Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Status:** Implemented; checkboxes are not updated. + +**Goal:** Store The Trade Desk ID (TDID) from Prebid's Unified ID first-party cookie, as an `[[ec.identity_providers]]` entry with `kind = "unifiedid"`. No server call. + +**Architecture:** `crates/trusted-server-identity-providers/src/unifiedid.rs` holds `UnifiedId` (a `ProviderKind`), `UnifiedIdSettings` and `UnifiedIdProvider`. The provider implements only `IdentityProvider::read_request_cookies` and keeps EID-level fields (`EidLevelFields::Preserved`). + +**Spec:** `docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md`, section 7. + +## Cookie + +- Prebid's `unifiedIdSystem` stores percent-encoded JSON in `pbjs-unifiedid`: `TDID`, `TDID_LOOKUP`, `TDID_CREATED_AT`. Only `TDID` is read. +- Default `cookie_names = ["pbjs-unifiedid"]`, read in order. +- Malformed JSON, a missing or non-string `TDID`, or a blank or opt-out ID reads as no ID. + +No Lockr capability. + +## Stored ID + +- One EID shaped as Prebid sends it: `atype = 1`, UID `ext` `{"rtiPartner":"TDID"}`, `inserter` and `matcher` `adserver.org`, `mm = 4`. +- Default bounded expiry; a cookie repeating the held ID is not rewritten, so the expiry renews only when the source is due. +- The source must not also be claimed by a liveintent `thetradedesk` attribute; pick one owner. + +## Not ported + +- The Trade Desk has no public publisher S2S ID API; only the Prebid cookie is read. + +## Tasks + +- [ ] Add `unifiedid.rs`; register the kind in `is_registered_kind` and `build`. +- [ ] Unit tests: TDID with Prebid's EID fields, missing, malformed and blank IDs ignored. +- [ ] Add a commented `kind = "unifiedid"` example to `trusted-server.example.toml`. +- [ ] Run `cargo test-fastly` and `cargo clippy-fastly`. diff --git a/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md b/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md index a955b2b46..936574a10 100644 --- a/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md +++ b/docs/superpowers/specs/2026-10-30-s2s-eid-collection-design.md @@ -1267,12 +1267,15 @@ For each provider with a `Found` outcome and each source in its response: unowned ones (D8). Each written UID records `owner` and `exp`. - Sources the provider claims but did not return stay unchanged. - `NoMatch`, failures, timeouts and skipped providers leave the row unchanged. -- `OptedOut` (a cookie or a refresh reply) clears every source the provider - claims and records the opt-out in the row's `opt_outs` map for 30 days. Until - it lapses the provider is skipped as `skipped_opted_out` and not called, and - a result from a lookup dispatched before the opt-out was recorded is - discarded rather than written over it. Generated and cookie IDs merge like - `Found`. +- `OptedOut` (a cookie or a vendor server's reply) clears every source the + provider claims. Only an opt-out a vendor server reports is recorded in the + row's `opt_outs` map for 30 days: a cookie opt-out is re-read on every + navigation, so it needs no memory. Until a recorded opt-out lapses the + provider is skipped as `skipped_opted_out` and not called, and a result from + a lookup dispatched before the opt-out was recorded is discarded rather than + written over it. The vendor's cookies are read before the recorded opt-out + is checked: an ID in them means the visitor opted back in, so the opt-out is + forgotten and the ID stored. Generated and cookie IDs merge like `Found`. Browser ingest (§8.4) is applied in the same in-memory merge, before provider results, so the write order never lets browser data overwrite provider data. @@ -1782,3 +1785,37 @@ combinations and adapter differences, on the tree after round 5's fix. - Non-blocking, no change: only the Fastly adapter builds the registry with provider ownership and vendor gates (§12.3 "On Fastly"); per-adapter deploy validation stays with #1552. + +### Round 7 (2026-11-12, PR #1488 at 5d22274c1, Claude Code (Opus 5.5) review agent) + +Angle: data flow and edge cases end to end per provider (UID2, ID5, +LiveRamp, ConnectID, LiveIntent, SharedID, Criteo, Lockr): cookie parsing, +request building, reply parsing, opt-out, 204 and 404 handling, refresh, +expiry, and consent and region gating. + +- No blocking findings. +- Non-blocking, fixed in 371e0ccba: LiveIntent kept a blank or `0` ID from + the reply; it now uses the shared usable-ID check. A LiveRamp envelope + entry without `type` failed the whole reply; it is now skipped. +- Non-blocking, no change: Lockr's storage format was checked against + Prebid's `lockrAIMIdSystem`. It writes only the raw `advertising_token` + under `key_name` to a cookie, with no `inserter` and no revoke marker, so + neither can be read from cookies. +- Known, open: LiveIntent partner UIDs carry `ext.provider` of + `liveintent.com` rather than the page domain, and a reply's `m` is not + copied into the `nonId` EID's `ext`. + +### Round 8 (2026-11-12, PR #1488 at 371e0ccba, Claude Code (Opus 5.5) review agent) + +Angle: adversarial. Malformed cookies and replies, config combinations, +clock edges and privacy leaks. + +- No blocking findings. Re-checked non-integer, negative, huge and NaN + expiry and TTL values (rejected, clamped to 30 days in validation, or + read as expired); empty and duplicate cookies, a stale opt-out cookie + beside a live ID (the opt-out wins); Lockr names equal to own names, Lockr + with empty own cookies, and Lockr under GDPR; and that no cookie value or + token is logged. +- Non-blocking, no change: a UID2 refresh that keeps failing is retried on + every navigation until the token expires unless `rate_limit_per_hour` is + set. The fix for overdue UID2 refresh state (834af2591) chose this bound. diff --git a/trusted-server.example.toml b/trusted-server.example.toml index a2c6dbbda..37b5a2d81 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -114,9 +114,7 @@ pull_sync_concurrency = 3 # pull_sync_enabled = false # default false # Optional server-side identity providers (Fastly only), looked up on publisher -# navigations before the origin fetch. Their budgets are set in [ec] above. This -# release registers no provider kind, so `ts config push` rejects an enabled -# provider and every runtime refuses to start with one. +# navigations before the origin fetch. Their budgets are set in [ec] above. # # [[ec.identity_providers]] # id = "example-provider" @@ -129,19 +127,211 @@ pull_sync_concurrency = 3 # must be claimed by only one enabled provider. # sources = ["ids.example.com"] # Optional: require TCF vendor consent for lookups and for sending any ID of -# these sources to bidders, including browser IDs, even while disabled. +# these sources to bidders, including browser IDs, even while disabled. Set it +# to the vendor's IAB Global Vendor List ID for every kind whose vendor is +# registered there; without it, GDPR traffic is gated only by the purpose +# consents. # tcf_vendor_id = 9999 # "page" (default) collects before the response; "after_response" collects -# after it is sent, only for an EC issued on an earlier request. +# after it is sent, only for an EC issued on an earlier request. Keep +# cookie-only kinds on "page". # collect = "page" # Optional: most lookups per EC hash (client IP) per hour. On Fastly this is # enforced per minute with a minimum of one, so values below 60 behave like 60. +# Useful for kinds whose unresolved sources stay due, such as LiveIntent +# partner IDs. # rate_limit_per_hour = 60 # Secret Store key name, resolved like other secret references. # [ec.identity_providers.secrets] # token = "example_provider_token" # Provider-specific settings, parsed by the provider kind. # [ec.identity_providers.settings] +# +# ID5 (kind "id5"): anonymous lookup, no secrets. The first source receives +# the legacy `universal_uid`. Use the endpoint and sources ID5 gives you. +# Before any lookup it reads the first-party cookies in `cookie_names`, in +# order. Prebid's ID5 module writes the `id5id` cookie only with +# `storage.type: 'cookie'`; with `html5` storage the lookup runs. +# +# Lockr capability (id5, liveramp and uid2 only): Lockr AIM writes other +# vendors' raw tokens into first-party cookies under key names configured per +# publisher in Lockr. Enable `[ec.identity_providers.settings.lockr]` to read +# them after the vendor's own cookies and before any lookup. Lockr cookies are +# not read where GDPR applies, because Lockr AIM does not run there. The +# default Lockr key of each kind equals one of its own cookie names, which are +# already read, so the capability changes nothing until `cookie_names` lists +# the key configured in Lockr when it differs. connectid and liveintent reject +# a `lockr` table. +# [[ec.identity_providers]] +# id = "id5" +# kind = "id5" +# enabled = false +# endpoint = "https://id5.example.com/gs/v2" +# allowed_domains = ["id5.example.com"] +# sources = ["id5.example.com"] +# tcf_vendor_id = 9999 +# [ec.identity_providers.settings] +# partner_id = 1234 +# Optional: who operates this integration, sent to ID5 as `provider`. +# Defaults to the publisher domain. +# provider_name = "publisher.example.com" +# cookie_names = ["id5id"] +# [ec.identity_providers.settings.lockr] +# enabled = true +# cookie_names = ["example_lockr_id5"] +# +# LiveIntent (kind "liveintent"): no hashed email, no secrets. Sends the +# visitor's first-party LiveIntent cookie (the first present one of +# `cookie_names`, default `_lc2_fpi`) as `duid`. `resolve` lists the attributes +# to ask for (default `nonId`); each one is stored under LiveIntent's fixed +# source for it, which must be listed in `sources`: nonId liveintent.com, +# bidswitch bidswitch.net, medianet media.net, magnite rubiconproject.com, +# index liveintent.indexexchange.com, openx openx.net, pubmatic pubmatic.com, +# sovrn liveintent.sovrn.com, sharethrough sharethrough.com, sonobi +# liveintent.sonobi.com, triplelift liveintent.triplelift.com, zetassp +# zeta-ssp.liveintent.com, vidazoo liveintent.vidazoo.com, nexxen +# liveintent.unrulymedia.com, uid2 uidapi.com, thetradedesk adserver.org. +# Where GDPR applies, uid2 is neither resolved nor kept, as for kind "uid2". +# `nonId` is LiveIntent's own ID and carries the reply's `segments` as EID +# `ext`; the others are partner IDs, sent with UID `ext.provider` naming +# LiveIntent (the Trade Desk ID also with `rtiPartner` and EID `matcher`). +# `distributor_id` must be a LiveIntent-issued `did-...` ID; without it the +# lookup uses the `prebid` path, as Prebid does. Set it to the publisher's own +# distributor ID (its Prebid `liveIntentId` `distributorId`) to get partner +# IDs: on the `prebid` path LiveIntent returns only `nonId`. +# [[ec.identity_providers]] +# id = "liveintent" +# kind = "liveintent" +# enabled = false +# endpoint = "https://liveintent.example.com/idex" +# allowed_domains = ["liveintent.example.com"] +# sources = ["liveintent.com", "rubiconproject.com"] +# tcf_vendor_id = 9999 +# [ec.identity_providers.settings] +# publisher_id = "any" +# distributor_id = "did-0123" +# resolve = ["nonId", "magnite"] +# cookie_names = ["_lc2_fpi"] +# +# LiveRamp (kind "liveramp"): reads the RampID envelope from LiveRamp's own +# first-party `_lr_env` cookie (ATS) or the bare envelope Prebid's +# identityLink module stores in `idl_env`, and keeps it for 15 days. Takes +# exactly one source; use the one LiveRamp gives you. Supports the Lockr +# capability. Optional server lookup: with an `endpoint` (LiveRamp's ATS +# envelope API, `/api/identity/v2/envelope`), a visitor without an envelope +# cookie is looked up by Custom ID: the stable part of the EC ID, sent as +# `:` with the visitor IP and consent. LiveRamp must +# enable Custom ID for `placement_id` and approve the publisher origin; both +# settings are required with an endpoint. LiveRamp resolves a Custom ID only +# through a mapping file the publisher sends it, tying each ID to a hashed +# email; without one the lookup finds nothing. Trusted Server itself sends no +# login or hashed email. +# [[ec.identity_providers]] +# id = "liveramp" +# kind = "liveramp" +# enabled = false +# endpoint = "https://liveramp.example.com/api/identity/v2/envelope" +# allowed_domains = ["liveramp.example.com"] +# sources = ["liveramp.example.com"] +# tcf_vendor_id = 9999 +# [ec.identity_providers.settings] +# cookie_names = ["_lr_env", "idl_env"] +# placement_id = 14 +# custom_id_namespace = "ana-example" +# [ec.identity_providers.settings.lockr] +# enabled = true +# cookie_names = ["example_lockr_lr_env"] +# +# Yahoo ConnectID (kind "connectid"): no lookup and no endpoint. Reads +# ConnectID from Yahoo's own first-party `connectId` cookie and keeps it until +# its last sync plus TTL (at most 14 days). Takes exactly one source. +# [[ec.identity_providers]] +# id = "connectid" +# kind = "connectid" +# enabled = false +# sources = ["connectid.example.com"] +# tcf_vendor_id = 9999 +# [ec.identity_providers.settings] +# cookie_names = ["connectId"] +# +# UID2 (kind "uid2"): reads the advertising token from the UID2 SDK's +# `__uid_2` cookie until its `identity_expires`, falling back to Prebid's raw +# `__uid2_advertising_token` cookie. Both clients keep the token in +# localStorage by default, which the server never sees: set the SDK's +# `useCookie: true` or Prebid's `params.storage: 'cookie'`. Takes exactly one +# source. `cookie_names` lists the raw token cookies, read in order after +# `identity_cookie_name`. +# Supports the Lockr capability; Lockr cookies are read last, as raw tokens. +# Optional server refresh: set `endpoint` to the UID2 operator's +# `/v2/token/refresh` URL. The `__uid_2` cookie's refresh token and response +# key are then kept privately in the row (never served), and the token is +# refreshed server side from `refresh_from`, with no API key and no email. +# Tokens are never generated: that needs an email or phone number. Without +# `endpoint`, the refresh token is never stored. The authenticated +# `/_ts/admin/ec/{id}` view shows refresh state as "[redacted]". UID2 is not +# offered where GDPR applies: for a visitor whose detected jurisdiction is +# GDPR it is skipped and the UIDs the row holds for it are removed. A TCF +# string alone does not count, since some CMPs set one for every visitor. +# [[ec.identity_providers]] +# id = "uid2" +# kind = "uid2" +# enabled = false +# endpoint = "https://uid2.example.com/v2/token/refresh" +# allowed_domains = ["uid2.example.com"] +# sources = ["uid2.example.com"] +# [ec.identity_providers.settings] +# identity_cookie_name = "__uid_2" +# cookie_names = ["__uid2_advertising_token"] +# [ec.identity_providers.settings.lockr] +# enabled = true +# cookie_names = ["example_lockr_uid2"] +# +# Prebid SharedID (kind "sharedid"): no lookup and no endpoint. Reads the +# publisher common ID from the cookie named in +# [[integrations.prebid.managed_user_ids]] `storage.name` for the `sharedId` +# module, or else from `_sharedid`, falling back to the legacy `_pubcid`, and +# sends it as a device-based (atype 1) ID. `sources` defaults to +# ["pubcid.org"], which needs a matching [[ec.partners]] entry. A stored +# SharedID lives as long as the row and wins over a different cookie value. +# The `_pubcid_optout` cookie suppresses it. With generate = true, when no +# cookie holds an ID and none is stored, Trusted Server generates a random UUID +# and stores it in the row only (no cookie is set). Leave it off unless Prebid +# keeps its opt-out in the `_pubcid_optout` cookie: a localStorage opt-out is +# invisible to the server. SharedID is the publisher's own ID, so it has no TCF +# vendor. +# [[ec.identity_providers]] +# id = "sharedid" +# kind = "sharedid" +# enabled = false +# [ec.identity_providers.settings] +# generate = false +# +# Criteo (kind "criteo"): no lookup and no endpoint. Reads Criteo's bid ID +# from the first-party `cto_bidid` cookie Prebid's Criteo module keeps and +# stores it percent-decoded, as Prebid sends it, as a device-based (atype 1) +# ID. Takes exactly one source. +# [[ec.identity_providers]] +# id = "criteo" +# kind = "criteo" +# enabled = false +# sources = ["criteo.example.com"] +# tcf_vendor_id = 9999 +# [ec.identity_providers.settings] +# cookie_names = ["cto_bidid"] +# +# Unified ID (kind "unifiedid"): no lookup and no endpoint. Reads The Trade +# Desk ID from the `TDID` field of Prebid's `pbjs-unifiedid` cookie and stores +# it as Prebid sends it: atype 1, UID `ext` `{"rtiPartner":"TDID"}`, and +# `inserter`/`matcher` `adserver.org` with `mm` 4. Takes exactly one source; +# do not also list that source for a liveintent `thetradedesk` attribute. +# [[ec.identity_providers]] +# id = "unifiedid" +# kind = "unifiedid" +# enabled = false +# sources = ["unifiedid.example.com"] +# tcf_vendor_id = 9999 +# [ec.identity_providers.settings] +# cookie_names = ["pbjs-unifiedid"] # =============================================================================