From 970acdf04beb727aa10fe9fe4b2788bdc4db8d19 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 4 Nov 2026 19:40:33 +0530 Subject: [PATCH 1/5] Fall back locally after mediator HTTP errors --- .../src/auction/orchestrator.rs | 177 +++++++++++++++++- .../src/integrations/adserver_mock.rs | 8 +- docs/guide/auction-orchestration.md | 1 + docs/guide/integrations/adserver_mock.md | 6 + 4 files changed, 187 insertions(+), 5 deletions(-) diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index f665441723..7a3cdb987e 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -756,7 +756,7 @@ impl AuctionOrchestratorHarness { .await .map_err(|error| { log::warn!( - "Planned mediator '{}' parse failed: {:?}", + "Planned mediator '{}' response processing failed; using local ranking: {:?}", mediator.provider_name(), error ); @@ -2648,7 +2648,7 @@ impl AuctionOrchestrator { } Err(error) => { log::warn!( - "Mediator '{}' parse failed: {:?}", + "Mediator '{}' response processing failed; using local ranking: {:?}", mediator.provider_name(), error ); @@ -2798,11 +2798,16 @@ mod tests { AuctionProvider, GenericOpenRtbProvider, ProviderRequestOutcome, }; use crate::auction::routing::{RoutingDiagnostics, route_auction}; + use crate::auction::telemetry::{ + AuctionObservationContext, AuctionSource, AuctionTerminalOutcome, build_auction_events, + }; use crate::auction::test_support::create_test_auction_context; use crate::auction::types::{ AdFormat, AdSlot, ApsRendererV1, ApsTagType, AuctionContext, AuctionRequest, AuctionResponse, Bid, BidRenderer, BidStatus, MediaType, PublisherInfo, UserInfo, }; + use crate::consent::types::ConsentContext; + use crate::ec::EcContext; use crate::error::TrustedServerError; use crate::integrations::adserver_mock::{AdServerMockConfig, AdServerMockProvider}; use crate::platform::test_support::{ @@ -7144,6 +7149,174 @@ mod tests { } } + async fn run_mock_mediator_response( + harness: bool, + status: u16, + body: &serde_json::Value, + floor: f64, + ) -> (AuctionRequest, OrchestrationResult) { + let http = Arc::new(StubHttpClient::new()); + http.push_response( + 200, + serde_json::to_vec(&serde_json::json!({ + "seatbid": [{"seat": "provider-seat", "bid": [ + {"id": "lower", "impid": "fictional-slot", "price": 1.5, + "adm": "
lower
", "w": 300, "h": 250}, + {"id": "higher", "impid": "fictional-slot", "price": 2.0, + "adm": "
higher
", "w": 300, "h": 250} + ]}] + })) + .expect("should serialize provider bids"), + ); + http.push_response( + status, + serde_json::to_vec(body).expect("should serialize mediator response"), + ); + let backend = Arc::new(NamingBackend::new(BackendNamingPolicy::Axum)); + let services = build_services_with_backend_and_http_client( + Arc::clone(&backend) as Arc<_>, + Arc::clone(&http) as Arc<_>, + ); + let plan = AuctionPlan::compile(planned_config( + &[("provider-a", RoutingMode::AllEligible)], + false, + )) + .expect("should compile planned auction"); + let mediator = Arc::new(AdServerMockProvider::new(AdServerMockConfig { + enabled: true, + endpoint: "https://mediator.example/mediate".to_string(), + timeout_ms: 500, + ..AdServerMockConfig::default() + })); + let mut request = planned_request(); + request.slots[0].floor_price = Some(floor); + let settings = create_test_settings(); + let inbound = http::Request::new(edgezero_core::body::Body::empty()); + let context = AuctionContext { + settings: &settings, + request: &inbound, + timeout_ms: 777, + transport_timeout_ms: 777, + provider_responses: None, + services: &services, + }; + + let result = if harness { + AuctionOrchestratorHarness::new(plan, Some(mediator)) + .run_auction(&request, &context) + .await + } else { + AuctionOrchestrator::from_plan(Arc::new(plan), Some(mediator)) + .run_auction(&request, &context) + .await + } + .expect("should complete auction with mock mediation"); + + (request, result) + } + + async fn assert_mediator_non_success_falls_back(harness: bool) { + for status in [302, 400, 503] { + for floor in [1.0, 3.0] { + let (request, result) = run_mock_mediator_response( + harness, + status, + &serde_json::json!({"seatbid": []}), + floor, + ) + .await; + let expected_winner = if floor < 2.0 { Some("higher") } else { None }; + assert_eq!( + result + .winning_bids + .get("fictional-slot") + .and_then(|bid| bid.bid_id.as_deref()), + expected_winner, + "should rank provider bids locally with floors after HTTP {status} (harness={harness})" + ); + assert_eq!( + result.provider_responses[0].bids.len(), + 2, + "should retain both collected provider bids" + ); + assert!( + result.mediator_response.is_none(), + "should treat HTTP {status} as failed mediation" + ); + + let ec_context = EcContext::new_for_test(None, ConsentContext::default()); + let observation = AuctionObservationContext::from_auction_request( + AuctionSource::AuctionApi, + &request, + &ec_context, + ); + let events = build_auction_events( + observation, + AuctionTerminalOutcome::Completed { + request: &request, + result: &result, + delivered_winner_slots: None, + }, + ); + assert!( + !events + .rows() + .iter() + .any(|row| row.provider_role.as_deref() == Some("mediator")), + "should omit failed mediator rows consistently with other mediation failures" + ); + assert_eq!( + events + .rows() + .iter() + .find(|row| row.event_kind == "summary") + .and_then(|row| row.winning_bid_count), + Some(u16::from(expected_winner.is_some())), + "should count locally selected winners in telemetry" + ); + } + } + } + + #[tokio::test] + async fn mediator_non_success_status_falls_back_to_local_ranking() { + assert_mediator_non_success_falls_back(false).await; + } + + #[tokio::test] + async fn planned_executor_mediator_non_success_status_falls_back_locally() { + assert_mediator_non_success_falls_back(true).await; + } + + #[tokio::test] + async fn mock_mediator_success_and_no_bid_remain_authoritative() { + for harness in [false, true] { + for has_winner in [false, true] { + let body = if has_winner { + serde_json::json!({"seatbid": [{"seat": "provider-a", "bid": [{ + "id": "mediated", "impid": "fictional-slot", "price": 1.75, + "adm": "
mediated
", "w": 300, "h": 250 + }]}]}) + } else { + serde_json::json!({"seatbid": []}) + }; + let (_, result) = run_mock_mediator_response(harness, 200, &body, 1.0).await; + assert!( + result.mediator_response.is_some(), + "should preserve a valid mediation result, including no-bid" + ); + assert_eq!( + result + .winning_bids + .get("fictional-slot") + .and_then(|bid| bid.bid_id.as_deref()), + if has_winner { Some("mediated") } else { None }, + "should honor the mediator decision instead of the higher local bid" + ); + } + } + } + #[tokio::test] async fn planned_executor_mediator_transport_failure_falls_back_locally() { let http = Arc::new(StubHttpClient::new()); diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/trusted-server-core/src/integrations/adserver_mock.rs index efc789f3aa..acff8a74eb 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/trusted-server-core/src/integrations/adserver_mock.rs @@ -377,7 +377,8 @@ impl AdServerMockProvider { /// /// # Errors /// - /// Returns an error when the mediation response body is not valid JSON. + /// Returns an error when the mediation response has a non-success HTTP status, + /// its body cannot be read within the response limit, or it is not valid JSON. async fn parse_response_inner( &self, response: PlatformResponse, @@ -388,8 +389,9 @@ impl AdServerMockProvider { let response = response.response; if !response.status().is_success() { - log::warn!("AdServer Mock returned non-success: {}", response.status()); - return Ok(AuctionResponse::error("adserver_mock", response_time_ms)); + return Err(Report::new(TrustedServerError::Auction { + message: format!("AdServer Mock returned HTTP {}", response.status()), + })); } // collect_response_bounded caps memory from misbehaving providers. diff --git a/docs/guide/auction-orchestration.md b/docs/guide/auction-orchestration.md index c377157ce5..f3d7841527 100644 --- a/docs/guide/auction-orchestration.md +++ b/docs/guide/auction-orchestration.md @@ -916,6 +916,7 @@ The orchestrator is designed to be resilient: - **Provider parse failure** — If a response can't be parsed, an `AuctionResponse::error()` is recorded. Other results are unaffected. - **No providers configured** — Completes as a no-bid without provider I/O. - **No provider produces a valid bid** — Returns an empty `OrchestrationResult` with zero winning bids after recording provider outcomes. +- **Mediator failure** — If mediation cannot run, including an HTTP non-2xx response, the orchestrator falls back to local ranking of collected provider bids with floor prices enforced. HTTP status and response parsing failures log a warning. As with other mediation failures, no mediator response or mediator telemetry row is recorded; the summary counts the locally selected winners. A valid successful no-bid response remains authoritative and does not trigger fallback. - **Mediator returns bids without decoded prices** — Those bids are filtered out with a warning. ## Observability diff --git a/docs/guide/integrations/adserver_mock.md b/docs/guide/integrations/adserver_mock.md index cc3fa44441..89d3a15cda 100644 --- a/docs/guide/integrations/adserver_mock.md +++ b/docs/guide/integrations/adserver_mock.md @@ -10,6 +10,12 @@ when both conditions hold. The orchestrator first gathers responses from the configured auction providers, then passes successful bids to the mediator's HTTP endpoint for final selection. +If the mediator request fails, returns a non-2xx HTTP status, or cannot be parsed, +the orchestrator warns and falls back to local ranking of the collected provider +bids, enforcing floor prices. A valid successful no-bid response remains final. +Failed mediation produces no mediator response or mediator telemetry row, matching +other mediation failures. + The mediator supports banner bids. It omits bids without a decoded numeric price, applies the optional CPM floor, and restores render and accounting fields from the original provider bids after the mediation response. Duplicate From 0894cb04184f0f1a867a31c2d34776fc47fca620 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 5 Nov 2026 11:17:43 +0530 Subject: [PATCH 2/5] Correct local winner telemetry after fallback Match local winning bids by instance ID and price so an earlier bid from the same bidder cannot claim the win. Preserve mediated association when mediation rewrites IDs or supplies decoded prices. Cover missing and reused IDs, provider collisions, ties, and HTTP-error fallback telemetry. Use level-neutral wording for mediator failure logs. --- .../src/auction/orchestrator.rs | 14 +++ .../src/auction/telemetry.rs | 116 +++++++++++++++++- docs/guide/integrations/adserver_mock.md | 2 +- 3 files changed, 129 insertions(+), 3 deletions(-) diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 7a3cdb987e..056a03b085 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -7274,6 +7274,20 @@ mod tests { Some(u16::from(expected_winner.is_some())), "should count locally selected winners in telemetry" ); + let bid_rows = events + .rows() + .iter() + .filter(|row| row.event_kind == "bid") + .map(|row| (row.price_cpm, row.is_win)) + .collect::>(); + assert_eq!( + bid_rows, + vec![ + (Some(1.5), Some(0)), + (Some(2.0), Some(u8::from(expected_winner.is_some()))), + ], + "should mark only the selected provider bid as winning after HTTP {status} (harness={harness})" + ); } } } diff --git a/crates/trusted-server-core/src/auction/telemetry.rs b/crates/trusted-server-core/src/auction/telemetry.rs index e0a819ba38..109f28813a 100644 --- a/crates/trusted-server-core/src/auction/telemetry.rs +++ b/crates/trusted-server-core/src/auction/telemetry.rs @@ -755,6 +755,10 @@ fn push_bid_rows( delivered_winner_slots.is_none_or(|slots| slots.contains(*slot_id)) && !matched_wins.contains(*slot_id) && bid_matches_winning_bid(bid, winning) + // Local ranking clones the selected bid without changing its identity or price. + // Mediation can replace both, so keep its original provider association. + && (result.mediator_response.is_some() + || (bid.bid_id == winning.bid_id && bid.price == winning.price)) }) .map(|(slot_id, winning)| (slot_id.clone(), winning)); let (is_win, price) = if let Some((slot_id, winning)) = matched_slot { @@ -1451,12 +1455,120 @@ mod tests { ); } + #[test] + fn local_winner_rows_identify_the_selected_bid() { + for (lower_id, higher_id) in [ + (None, None), + (Some("lower"), Some("higher")), + (Some("shared"), Some("shared")), + ] { + for ad_id in [None, Some("shared-ad")] { + for separate_providers in [false, true] { + let request = test_request("local-auction"); + let mut lower = bid("slot-1", "example-bidder", ad_id, Some(1.5)); + lower.bid_id = lower_id.map(String::from); + let mut higher = bid("slot-1", "example-bidder", ad_id, Some(2.0)); + higher.bid_id = higher_id.map(String::from); + let winner = higher.clone(); + let provider_responses = if separate_providers { + vec![ + AuctionResponse::success("provider-a", vec![lower], 1), + AuctionResponse::success("provider-b", vec![higher], 2), + ] + } else { + vec![AuctionResponse::success( + "provider-a", + vec![lower, higher], + 2, + )] + }; + let result = OrchestrationResult { + provider_responses, + mediator_response: None, + winning_bids: HashMap::from([("slot-1".to_owned(), winner)]), + total_time_ms: 2, + metadata: HashMap::new(), + }; + let events = build_auction_events( + AuctionObservationContext::for_test( + AuctionSource::AuctionApi, + "/auction", + 1, + ), + AuctionTerminalOutcome::Completed { + request: &request, + result: &result, + delivered_winner_slots: None, + }, + ); + let rows = events + .rows() + .iter() + .filter(|row| row.event_kind == "bid") + .map(|row| (row.provider.as_deref(), row.price_cpm, row.is_win)) + .collect::>(); + let winning_provider = if separate_providers { + "provider-b" + } else { + "provider-a" + }; + assert_eq!( + rows, + vec![ + (Some("provider-a"), Some(1.5), Some(0)), + (Some(winning_provider), Some(2.0), Some(1)), + ], + "should attribute the win and CPM to the selected bid even with absent or reused IDs" + ); + } + } + } + } + + #[test] + fn local_tied_bids_keep_the_first_provider_as_the_only_winner() { + let request = test_request("tied-auction"); + let first = bid("slot-1", "example-bidder", None, Some(2.0)); + let second = first.clone(); + let result = OrchestrationResult { + winning_bids: HashMap::from([("slot-1".to_owned(), first.clone())]), + provider_responses: vec![ + AuctionResponse::success("provider-a", vec![first], 1), + AuctionResponse::success("provider-b", vec![second], 2), + ], + mediator_response: None, + total_time_ms: 2, + metadata: HashMap::new(), + }; + let events = build_auction_events( + AuctionObservationContext::for_test(AuctionSource::AuctionApi, "/auction", 1), + AuctionTerminalOutcome::Completed { + request: &request, + result: &result, + delivered_winner_slots: None, + }, + ); + let winning_providers = events + .rows() + .iter() + .filter(|row| row.event_kind == "bid" && row.is_win == Some(1)) + .map(|row| row.provider.as_deref()) + .collect::>(); + assert_eq!( + winning_providers, + vec![Some("provider-a")], + "should preserve local ranking's first-provider tie decision without duplicate wins" + ); + } + #[test] fn mediated_win_marks_original_bid_once() { let request = test_request("req"); - let original_bid = bid("slot-1", "kargo", Some("ad-1"), None); + let mut original_bid = bid("slot-1", "example-bidder", Some("ad-1"), None); + original_bid.bid_id = Some("original-bid".to_owned()); let provider_success = AuctionResponse::success("prebid", vec![original_bid], 42); - let mediator_bid = bid("slot-1", "kargo", Some("ad-1"), Some(2.0)); + let mut mediator_bid = bid("slot-1", "example-bidder", Some("ad-1"), Some(2.0)); + mediator_bid.bid_id = Some("mediated-bid".to_owned()); let mediator_response = AuctionResponse::success("adserver_mock", vec![mediator_bid.clone()], 15); let result = OrchestrationResult { diff --git a/docs/guide/integrations/adserver_mock.md b/docs/guide/integrations/adserver_mock.md index 89d3a15cda..77f5711cd6 100644 --- a/docs/guide/integrations/adserver_mock.md +++ b/docs/guide/integrations/adserver_mock.md @@ -11,7 +11,7 @@ configured auction providers, then passes successful bids to the mediator's HTTP endpoint for final selection. If the mediator request fails, returns a non-2xx HTTP status, or cannot be parsed, -the orchestrator warns and falls back to local ranking of the collected provider +the orchestrator logs and falls back to local ranking of the collected provider bids, enforcing floor prices. A valid successful no-bid response remains final. Failed mediation produces no mediator response or mediator telemetry row, matching other mediation failures. From fe9a220e6cb07955586c308698439c166cc15276 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 5 Nov 2026 20:03:31 +0530 Subject: [PATCH 3/5] Document local winner identity contract and keep mediator error wording State on select_winning_bids that winners are verbatim clones of provider bids, since telemetry attributes local wins by bid_id and price. Restore the "non-success" wording in the mediator HTTP error so existing log searches still match. --- crates/trusted-server-core/src/auction/orchestrator.rs | 5 +++++ crates/trusted-server-core/src/integrations/adserver_mock.rs | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 056a03b085..488468c62d 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -1570,6 +1570,11 @@ impl AuctionOrchestrator { } /// Select the best decoded-price bid for each slot from all responses. + /// + /// Winning bids are verbatim clones of provider bids, and floor enforcement + /// only removes entries. Auction telemetry attributes local wins by matching + /// `bid_id` and `price`, so any future change that rewrites a winner (for + /// example currency conversion or bid shading) must update that matching too. fn select_winning_bids( &self, responses: &[AuctionResponse], diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/trusted-server-core/src/integrations/adserver_mock.rs index acff8a74eb..ac3d7ba105 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/trusted-server-core/src/integrations/adserver_mock.rs @@ -390,7 +390,10 @@ impl AdServerMockProvider { if !response.status().is_success() { return Err(Report::new(TrustedServerError::Auction { - message: format!("AdServer Mock returned HTTP {}", response.status()), + message: format!( + "AdServer Mock returned non-success HTTP {}", + response.status() + ), })); } From 7ff0d45ed1a875d70287dba3bd594829d0461f13 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 11 Nov 2026 19:19:23 +0530 Subject: [PATCH 4/5] Address mediator fallback review feedback Attribute local telemetry wins to the exact winning provider bid instead of inferring local ranking from an absent mediator response, so keeping a failed mediator response cannot reattach the win to an earlier, lower bid. Treat a mediator HTTP 204 as a final no-bid, matching how bidder providers handle 204, and fall back to local ranking when a mediator reports an error-status response on both the production and harness paths. Clarify the mediator failure wording in the auction guide and add CHANGELOG entries for the fallback and telemetry changes. --- CHANGELOG.md | 2 + .../src/auction/orchestrator.rs | 132 ++++++++++++++++-- .../src/auction/telemetry.rs | 64 ++++++++- .../src/integrations/adserver_mock.rs | 8 +- docs/guide/auction-orchestration.md | 2 +- docs/guide/integrations/adserver_mock.md | 3 +- 6 files changed, 192 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa4e98e7ab..8cbd00f03e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- A mediator that answers with a non-2xx HTTP status no longer discards the collected provider bids. The orchestrator now logs a warning and ranks those bids locally with floors, as it already did for mediator launch, transport, deadline and parse failures (#1203). A mediator response reported with an error status falls back the same way. An HTTP 204 from the `adserver_mock` mediator is a final no-bid, matching how bidder providers treat 204. +- Auction telemetry now marks the bid that local ranking selected as the slot's winner. When one bidder code returned several bids for a slot without distinguishing ad IDs, an earlier lower-priced bid, possibly from another provider, was marked as the win at its own price. Tinybird per-provider win counts and winning-CPM quantiles for those slots shift after upgrading. Mediated wins keep their provider association unless the mediator returns a provider bid's own ID and price; that exact bid is then marked. - Lockr and Permutive SDK recognition now uses parsed vendor hostnames and SDK pathnames instead of searching the whole URL (#1199). The browser Lockr guard no longer replaces unrelated scripts that mention a Lockr host in their path or query, use a lookalike host, or serve a non-SDK resource from a Lockr host. Query strings and fragments no longer prevent SDK recognition in the server matchers, the browser Permutive guard, or the browser Lockr guard on `identity.loc.kr`. Matching URLs use the existing configured-SDK routes; their original query and fragment are not forwarded upstream. Server matching trims leading and trailing C0 controls and spaces, removes tab, CR, and LF, and accepts backslashes as authority separators. It no longer recognizes attribute values that contain a NUL or that write slashes as character references such as `/`, so markup like that can load the SDK from the vendor host. - `rewrite.exclude_domains` matching is now case-insensitive and ignores surrounding whitespace. Entries written with uppercase letters previously never matched and now take effect, so those hosts stop being proxied and click-wrapped, and `/first-party/sign` now rejects them with `502`; it used to sign them, or return `403` when the host was also outside `proxy.allowed_domains`. Empty and bare `"*"` `exclude_domains` entries, which never matched a host, are dropped at load with a warning, and other entries that can never match a host are logged with a warning. Absolute `http(s)` creative URLs that cannot be parsed (for example, a host containing a space) are now left untouched; previously the trimmed value was written back re-quoted (with `https:` prepended to `//` input, so `src=" //example.com invalid/x "` became `https://example.com invalid/x`), an unparseable quoted URL in `